<commit_before># coding=utf-8
"""
Blueprint for views of dynamic images
"""
from __future__ import absolute_import

import hashlib
from datetime import datetime, timedelta
from pathlib import Path
import colorsys

import sqlalchemy as sa
import pkg_resources

from flask import Blueprint, request, make_response, render_template
from werkzeug.exceptions import BadRequest, NotFound
from abilian.core.util import utc_dt
from abilian.core.models.blob import Blob
from abilian.core.models.subjects import User
from abilian.web.util import url_for

from .base import View

blueprint = Blueprint('images', __name__, url_prefix='/images')
route = blueprint.route

DEFAULT_AVATAR = Path(
    pkg_resources.resource_filename(
        'abilian.web',
        'resources/img/avatar-default.png')
)
DEFAULT_AVATAR_MD5 = hashlib.md5(DEFAULT_AVATAR.open('rb').read()).hexdigest()

class BaseImageView(View):
  """
  """
  max_size = None
  set_expire = False
  expire_offset = timedelta(days=365)

  #: argument name that must be found in view kwargs. This is a safety measure
  #: to prevent setting far expire date on resources without a varying argument
  #: in url (path or query string), such as a timestamp, a serial, a hash...
  expire_vary_arg = None

  def __init__(self, max_size=None, set_expire=None, expire_offset=None,
               expire_vary_arg=None):
    # override class default value only if arg is specified in constructor. This
    # allows subclasses to easily override theses defaults.
    if max_size is not None:
      self.max_size = max_size
    if set_expire is not None:
      self.set_expire = set_expire
    if expire_offset is not None:
      self.expire_offset = expire_offset
    if expire_vary_arg is not None:
      self.expire_vary_arg = expire_vary_arg

    if self.set_expire:
      if not self.expire_offset:
        raise ValueError('expire_offset is not set')
      if not self.expire_vary_arg:
        raise ValueError('expire_vary_arg is not set')

  def prepare_args(self, args, kwargs):
    if self.set_expire:
      vary_arg = kwargs.get(self.expire_vary_arg,
                            request.args.get(self.expire_vary_arg, None))
      if vary_arg is None:
        # argument for timestamp, serial etc is missing. We must refuse to serve
        # an image with expiry date set up to maybe 1 year from now.
        # Check the code that has generated this url!
        raise BadRequest(
          'Image version marker is missing ({}=?)'.format(
            repr(self.expire_vary_arg))
        )

    args, kwargs = View.prepare_args(self, args, kwargs)
    size = request.args.get('s', 0)
    try:
      size = int(size)
    except ValueError:
      raise BadRequest(
        'Invalid value for "s": {}. Not an integer.'.format(repr(size))
      )

    if self.max_size is not None:
      if size > self.max_size:
        raise BadRequest(
          'Size too large: {:d} (max: {:d})'.format(size, self.max_size)
        )

    kwargs['size'] = size
    return args, kwargs

  def get(self, image, size, *args, **kwargs):
    """
    :param image: image as bytes
    :param s: requested maximum width/height size
    """
    from abilian.services.image import crop_and_resize, get_format

    try:
      fmt = get_format(image)
    except IOError:
      #  not a known image file
      raise NotFound()

    content_type = u'image/png' if fmt == 'PNG' else u'image/jpeg'

    if size:
      image = crop_and_resize(image, size)
    else:
      image = image.read()

    response = make_response(image)
    response.headers['content-type'] = content_type
    self.set_cache_headers(response)
    return response

  def set_cache_headers(self, response):
    """
    """
    if self.set_expire:
      response.cache_control.private = True
      response.cache_control.max_age = int(self.expire_offset.total_seconds())
      response.expires = utc_dt(datetime.utcnow() + self.expire_offset)

class StaticImageView(BaseImageView):
  """
  View for static assets not served by static directory.

  Useful for default avatars for example.
  """
  expire_vary_arg = 'md5'

  def __init__(self, image, *args, **kwargs):
    """
    :param image: path to image file
    """
    BaseImageView.__init__(self, *args, **kwargs)
    self.image_path = Path(image)
    if not self.image_path.exists():
      p = unicode(self.image_path)
      raise ValueError('Invalid image path: {}'.format(repr(p)))

  def prepare_args(self, args, kwargs):
    kwargs['image'] = self.image_path.open('rb')
    return BaseImageView.prepare_args(self, args, kwargs)

class BlobView(BaseImageView):
  """
  Default :attr:`expire_vary_arg` to `"md5"`.
  :attr:`set_expire` is set to `False` by default.
  """
  expire_vary_arg = 'md5'
  id_arg = 'object_id'

  def __init__(self, id_arg=None, *args, **kwargs):
    BaseImageView.__init__(self, *args, **kwargs)
    if id_arg is not None:
      self.id_arg = id_arg

  def prepare_args(self, args, kwargs):
    args, kwargs = BaseImageView.prepare_args(self, args, kwargs)
    b_id = kwargs.get(self.id_arg)
    try:
      b_id = int(b_id)
    except ValueError:
      raise BadRequest('Invalid image id: {}'.format(repr(b_id)))

    blob = Blob.query.get(b_id)
    if not blob:
      raise NotFound()

    kwargs['image'] = blob.file.open('rb')
    return args, kwargs

blob_image = BlobView.as_view('blob_image')
route("/files/<int:object_id>")(blob_image)

class UserMugshot(BaseImageView):
  expire_vary_arg = 'md5'

  def prepare_args(self, args, kwargs):
    args, kwargs = BaseImageView.prepare_args(self, args, kwargs)

    user_id = kwargs['user_id']
    user = User.query\
        .options(sa.orm.undefer(User.photo))\
        .get(user_id)

    if user is None:
      raise NotFound()

    kwargs['user'] = user
    kwargs['image'] = user.photo
    return args, kwargs

  def get(self, user, image, size, *args, **kwargs):
    if image:
      #  user has set a photo
      return super(UserMugshot, self).get(image, size, *args, **kwargs)

    # render svg avatar
    letter = user.last_name[0] if user.last_name else user.first_name[0]
    letter = letter.upper()
    # generate bg color, pastel: sat=65% in hsl color space
    id_hash = hash((user.name + user.email).encode('utf-8'))
    hue = id_hash % 10
    hue = (hue * 36) / 360.0  # 10 colors: 360 / 10
    color = colorsys.hsv_to_rgb(hue, 0.65, 1.0)
    color = [int(x * 255) for x in color]
    color = u'rgb({0[0]}, {0[1]}, {0[2]})'.format(color)
    svg = render_template('default/avatar.svg',
                          color=color, letter=letter, size=size)
    response = make_response(svg)
    response.headers['content-type'] = u'image/svg+xml'
    self.set_cache_headers(response)
    return response

user_photo = UserMugshot.as_view('user_photo', set_expire=True, max_size=500)
route("/users/<int:user_id>")(user_photo)
route('/users/default')(StaticImageView.as_view('user_default',
                                                set_expire=True,
                                                image=DEFAULT_AVATAR,))

def user_url_args(user, size):
  endpoint = 'images.user_default'
  kwargs = {'s': size,
            'md5': DEFAULT_AVATAR_MD5,}

  if not user.is_anonymous():
    endpoint = 'images.user_photo'
    kwargs['user_id'] = user.id
    content = (user.photo if user.photo
               else (user.name + user.email).encode('utf-8'))
    kwargs['md5'] = hashlib.md5(content).hexdigest()

  return endpoint, kwargs

def user_photo_url(user, size):
  """
  Return url to use for this user
  """
  endpoint, kwargs = user_url_args(user, size)
  return url_for(endpoint, **kwargs)
<commit_msg>Fix corner case.<commit_after># coding=utf-8
"""
Blueprint for views of dynamic images
"""
from __future__ import absolute_import

import hashlib
from datetime import datetime, timedelta
from pathlib import Path
import colorsys

import sqlalchemy as sa
import pkg_resources

from flask import Blueprint, request, make_response, render_template
from werkzeug.exceptions import BadRequest, NotFound
from abilian.core.util import utc_dt
from abilian.core.models.blob import Blob
from abilian.core.models.subjects import User
from abilian.web.util import url_for

from .base import View

blueprint = Blueprint('images', __name__, url_prefix='/images')
route = blueprint.route

DEFAULT_AVATAR = Path(
    pkg_resources.resource_filename(
        'abilian.web',
        'resources/img/avatar-default.png')
)
DEFAULT_AVATAR_MD5 = hashlib.md5(DEFAULT_AVATAR.open('rb').read()).hexdigest()

class BaseImageView(View):
  """
  """
  max_size = None
  set_expire = False
  expire_offset = timedelta(days=365)

  #: argument name that must be found in view kwargs. This is a safety measure
  #: to prevent setting far expire date on resources without a varying argument
  #: in url (path or query string), such as a timestamp, a serial, a hash...
  expire_vary_arg = None

  def __init__(self, max_size=None, set_expire=None, expire_offset=None,
               expire_vary_arg=None):
    # override class default value only if arg is specified in constructor. This
    # allows subclasses to easily override theses defaults.
    if max_size is not None:
      self.max_size = max_size
    if set_expire is not None:
      self.set_expire = set_expire
    if expire_offset is not None:
      self.expire_offset = expire_offset
    if expire_vary_arg is not None:
      self.expire_vary_arg = expire_vary_arg

    if self.set_expire:
      if not self.expire_offset:
        raise ValueError('expire_offset is not set')
      if not self.expire_vary_arg:
        raise ValueError('expire_vary_arg is not set')

  def prepare_args(self, args, kwargs):
    if self.set_expire:
      vary_arg = kwargs.get(self.expire_vary_arg,
                            request.args.get(self.expire_vary_arg, None))
      if vary_arg is None:
        # argument for timestamp, serial etc is missing. We must refuse to serve
        # an image with expiry date set up to maybe 1 year from now.
        # Check the code that has generated this url!
        raise BadRequest(
          'Image version marker is missing ({}=?)'.format(
            repr(self.expire_vary_arg))
        )

    args, kwargs = View.prepare_args(self, args, kwargs)
    size = request.args.get('s', 0)
    try:
      size = int(size)
    except ValueError:
      raise BadRequest(
        'Invalid value for "s": {}. Not an integer.'.format(repr(size))
      )

    if self.max_size is not None:
      if size > self.max_size:
        raise BadRequest(
          'Size too large: {:d} (max: {:d})'.format(size, self.max_size)
        )

    kwargs['size'] = size
    return args, kwargs

  def get(self, image, size, *args, **kwargs):
    """
    :param image: image as bytes
    :param s: requested maximum width/height size
    """
    from abilian.services.image import crop_and_resize, get_format

    try:
      fmt = get_format(image)
    except IOError:
      #  not a known image file
      raise NotFound()

    content_type = u'image/png' if fmt == 'PNG' else u'image/jpeg'

    if size:
      image = crop_and_resize(image, size)
    else:
      image = image.read()

    response = make_response(image)
    response.headers['content-type'] = content_type
    self.set_cache_headers(response)
    return response

  def set_cache_headers(self, response):
    """
    """
    if self.set_expire:
      response.cache_control.private = True
      response.cache_control.max_age = int(self.expire_offset.total_seconds())
      response.expires = utc_dt(datetime.utcnow() + self.expire_offset)

class StaticImageView(BaseImageView):
  """
  View for static assets not served by static directory.

  Useful for default avatars for example.
  """
  expire_vary_arg = 'md5'

  def __init__(self, image, *args, **kwargs):
    """
    :param image: path to image file
    """
    BaseImageView.__init__(self, *args, **kwargs)
    self.image_path = Path(image)
    if not self.image_path.exists():
      p = unicode(self.image_path)
      raise ValueError('Invalid image path: {}'.format(repr(p)))

  def prepare_args(self, args, kwargs):
    kwargs['image'] = self.image_path.open('rb')
    return BaseImageView.prepare_args(self, args, kwargs)

class BlobView(BaseImageView):
  """
  Default :attr:`expire_vary_arg` to `"md5"`.
  :attr:`set_expire` is set to `False` by default.
  """
  expire_vary_arg = 'md5'
  id_arg = 'object_id'

  def __init__(self, id_arg=None, *args, **kwargs):
    BaseImageView.__init__(self, *args, **kwargs)
    if id_arg is not None:
      self.id_arg = id_arg

  def prepare_args(self, args, kwargs):
    args, kwargs = BaseImageView.prepare_args(self, args, kwargs)
    b_id = kwargs.get(self.id_arg)
    try:
      b_id = int(b_id)
    except ValueError:
      raise BadRequest('Invalid image id: {}'.format(repr(b_id)))

    blob = Blob.query.get(b_id)
    if not blob:
      raise NotFound()

    kwargs['image'] = blob.file.open('rb')
    return args, kwargs

blob_image = BlobView.as_view('blob_image')
route("/files/<int:object_id>")(blob_image)

class UserMugshot(BaseImageView):
  expire_vary_arg = 'md5'

  def prepare_args(self, args, kwargs):
    args, kwargs = BaseImageView.prepare_args(self, args, kwargs)

    user_id = kwargs['user_id']
    user = User.query\
        .options(sa.orm.undefer(User.photo))\
        .get(user_id)

    if user is None:
      raise NotFound()

    kwargs['user'] = user
    kwargs['image'] = user.photo
    return args, kwargs

  def get(self, user, image, size, *args, **kwargs):
    if image:
      #  user has set a photo
      return super(UserMugshot, self).get(image, size, *args, **kwargs)

    # render svg avatar
    if user.last_name:
      letter = user.last_name[0]
    elif user.first_name:
      letter = user.first_name[0]
    else:
      letter = u"?"
    letter = letter.upper()
    # generate bg color, pastel: sat=65% in hsl color space
    id_hash = hash((user.name + user.email).encode('utf-8'))
    hue = id_hash % 10
    hue = (hue * 36) / 360.0  # 10 colors: 360 / 10
    color = colorsys.hsv_to_rgb(hue, 0.65, 1.0)
    color = [int(x * 255) for x in color]
    color = u'rgb({0[0]}, {0[1]}, {0[2]})'.format(color)
    svg = render_template('default/avatar.svg',
                          color=color, letter=letter, size=size)
    response = make_response(svg)
    response.headers['content-type'] = u'image/svg+xml'
    self.set_cache_headers(response)
    return response

user_photo = UserMugshot.as_view('user_photo', set_expire=True, max_size=500)
route("/users/<int:user_id>")(user_photo)
route('/users/default')(StaticImageView.as_view('user_default',
                                                set_expire=True,
                                                image=DEFAULT_AVATAR,))

def user_url_args(user, size):
  endpoint = 'images.user_default'
  kwargs = {'s': size,
            'md5': DEFAULT_AVATAR_MD5,}

  if not user.is_anonymous():
    endpoint = 'images.user_photo'
    kwargs['user_id'] = user.id
    content = (user.photo if user.photo
               else (user.name + user.email).encode('utf-8'))
    kwargs['md5'] = hashlib.md5(content).hexdigest()

  return endpoint, kwargs

def user_photo_url(user, size):
  """
  Return url to use for this user
  """
  endpoint, kwargs = user_url_args(user, size)
  return url_for(endpoint, **kwargs)
<|endoftext|>


<commit_before>import os
import sys
import time
import errno
import signal
import inspect
import subprocess
import cPickle as pickle
from abusehelper.core import bot, config

class Startup(config.Config):
    def startup(self):
        for key, value in self.member_diff(Startup):
            if not inspect.isroutine(value):
                yield key, value

def kill_processes(processes, sig):
    for process in processes:
        try:
            os.kill(process.pid, sig)
        except OSError, ose:
            if ose.errno != errno.ESRCH:
                raise

class StartupBot(bot.Bot):
    ini_file = None
    ini_section = None
    startup = None

    def configs(self):
        return []

    def run(self, poll_interval=0.1):
        def signal_handler(sig, frame):
            sys.exit()
        signal.signal(signal.SIGTERM, signal_handler)

        processes = dict()
        try:
            for startup in self.configs():
                if not hasattr(startup, "startup"):
                    continue
                params = dict(startup.startup())

                module = params["module"]
                bot_name = params["bot_name"]
                self.log.info("Launching bot %r from module %r",
                              bot_name, module)

                args = [sys.executable]
                path, _ = os.path.split(module)
                if path:
                    args.extend([module])
                else:
                    # At least Python 2.5 on OpenBSD replaces the
                    # argument right after the -m option with "-c" in
                    # the process listing, making it harder to figure
                    # out which modules are running. Workaround: Use
                    # "-m runpy module" instead of "-m module".
                    args.extend(["-m", "runpy", module])
                args.append("--startup")

                process = subprocess.Popen(args, stdin=subprocess.PIPE)
                processes[startup] = process

                pickle.dump(params, process.stdin)
                process.stdin.flush()

            while True:
                for startup, process in processes.items():
                    retval = process.poll()
                    if retval is not None:
                        return
                time.sleep(poll_interval)
        finally:
            processes = self.check_processes(processes)

            if processes:
                self.log.info("Sending SIGTERM to alive bots")
                kill_processes(processes.values(), signal.SIGTERM)

            while processes:
                processes = self.check_processes(processes)
                time.sleep(poll_interval)

    def check_processes(self, processes):
        processes = dict(processes)

        for startup, process in list(processes.items()):
            retval = process.poll()
            if retval is None:
                continue

            del processes[startup]
            self.log.info("Bot %r exited with return value %d",
                          startup.bot_name, retval)

        return processes

class DefaultStartupBot(StartupBot):
    config = bot.Param("configuration module")
    enable = bot.ListParam("bots that are run (default: run all bots)",
                           default=None)
    disable = bot.ListParam("bots that are not run (default: run all bots)",
                            default=None)

    def configs(self):
        for conf_obj in set(config.load_configs(os.path.abspath(self.config))):
            if self.disable is not None and conf_obj.name in self.disable:
                continue
            if self.enable is not None and conf_obj.name not in self.enable:
                continue
            yield conf_obj

if __name__ == "__main__":
    DefaultStartupBot.from_command_line().execute()
<commit_msg>Fixes for startup bot to not use the config object's attributes directly.<commit_after>import os
import sys
import time
import errno
import signal
import inspect
import subprocess
import cPickle as pickle
from abusehelper.core import bot, config

class Startup(config.Config):
    def startup(self):
        for key, value in self.member_diff(Startup):
            if not inspect.isroutine(value):
                yield key, value

def kill_processes(processes, sig):
    for process in processes:
        try:
            os.kill(process.pid, sig)
        except OSError, ose:
            if ose.errno != errno.ESRCH:
                raise

class StartupBot(bot.Bot):
    ini_file = None
    ini_section = None
    startup = None

    def configs(self):
        return []

    def run(self, poll_interval=0.1):
        def signal_handler(sig, frame):
            sys.exit()
        signal.signal(signal.SIGTERM, signal_handler)

        processes = dict()
        try:
            for startup in self.configs():
                if not hasattr(startup, "startup"):
                    continue
                params = dict(startup.startup())

                module = params["module"]
                name = params["bot_name"]
                self.log.info("Launching bot %r from module %r", name, module)

                args = [sys.executable]
                path, _ = os.path.split(module)
                if path:
                    args.extend([module])
                else:
                    # At least Python 2.5 on OpenBSD replaces the
                    # argument right after the -m option with "-c" in
                    # the process listing, making it harder to figure
                    # out which modules are running. Workaround: Use
                    # "-m runpy module" instead of "-m module".
                    args.extend(["-m", "runpy", module])
                args.append("--startup")

                process = subprocess.Popen(args, stdin=subprocess.PIPE)
                processes[startup] = name, process

                pickle.dump(params, process.stdin)
                process.stdin.flush()

            while True:
                for startup, (_, process) in processes.items():
                    retval = process.poll()
                    if retval is not None:
                        return
                time.sleep(poll_interval)
        finally:
            processes = self.check_processes(processes)

            if processes:
                self.log.info("Sending SIGTERM to alive bots")
                kill_processes([x[1] for x in processes.values()],
                               signal.SIGTERM)

            while processes:
                processes = self.check_processes(processes)
                time.sleep(poll_interval)

    def check_processes(self, processes):
        processes = dict(processes)

        for startup, (name, process) in list(processes.items()):
            retval = process.poll()
            if retval is None:
                continue

            del processes[startup]
            self.log.info("Bot %r exited with return value %d", name, retval)

        return processes

class DefaultStartupBot(StartupBot):
    config = bot.Param("configuration module")
    enable = bot.ListParam("bots that are run (default: run all bots)",
                           default=None)
    disable = bot.ListParam("bots that are not run (default: run all bots)",
                            default=None)

    def configs(self):
        for conf_obj in set(config.load_configs(os.path.abspath(self.config))):
            if self.disable is not None and conf_obj.name in self.disable:
                continue
            if self.enable is not None and conf_obj.name not in self.enable:
                continue
            yield conf_obj

if __name__ == "__main__":
    DefaultStartupBot.from_command_line().execute()
<|endoftext|>


<commit_before>#!/usr/bin/env python

from tables import *
import numpy as NA
import struct
import sys
import shelve
import psyco

# This class is accessible only for the examples

class Small(IsDescription):

    """Record descriptor.

    A record has several columns. They are represented here as class
    attributes, whose names are the column names and their values will
    become their types. The IsDescription class will take care the user
    will not add any new variables and that its type is correct.

    """

    var1 = StringCol(itemsize=4)
    var2 = Int32Col()
    var3 = Float64Col()

# Define a user record to characterize some kind of particles

class Medium(IsDescription):
    name = StringCol(itemsize=16)   # 16-character String
    float1 = Float64Col(shape=2, dflt=2.3)
    #float1 = Float64Col(dflt=1.3)
    #float2 = Float64Col(dflt=2.3)
    ADCcount = Int16Col()           # signed short integer
    grid_i = Int32Col()             # integer
    grid_j = Int32Col()             # integer
    pressure = Float32Col()         # float  (single-precision)
    energy = Float64Col()           # double (double-precision)

# Define a user record to characterize some kind of particles

class Big(IsDescription):
    name = StringCol(itemsize=16)   # 16-character String
    #float1 = Float64Col(shape=32, dflt=NA.arange(32))
    #float2 = Float64Col(shape=32, dflt=NA.arange(32))
    float1 = Float64Col(shape=32, dflt=range(32))
    float2 = Float64Col(shape=32, dflt=[2.2] * 32)
    ADCcount = Int16Col()           # signed short integer
    grid_i = Int32Col()             # integer
    grid_j = Int32Col()             # integer
    pressure = Float32Col()         # float  (single-precision)
    energy = Float64Col()           # double (double-precision)

def createFile(filename, totalrows, recsize):

    # Open a 'n'ew file
    fileh = shelve.open(filename, flag="n")

    rowswritten = 0
    # Get the record object associated with the new table
    if recsize == "big":
        d = Big()
        arr = NA.array(NA.arange(32), type=NA.float64)
        arr2 = NA.array(NA.arange(32), type=NA.float64)
    elif recsize == "medium":
        d = Medium()
    else:
        d = Small()
    # print d
    # sys.exit(0)
    for j in range(3):
        # Create a table
        # table = fileh.create_table(group, 'tuple'+str(j), Record(), title,
        #                          compress = 6, expectedrows = totalrows)
        # Create a Table instance
        tablename = 'tuple' + str(j)
        table = []
        # Fill the table
        if recsize == "big" or recsize == "medium":
            for i in range(totalrows):
                d.name = 'Particle: %6d' % (i)
                #d.TDCcount = i % 256
                d.ADCcount = (i * 256) % (1 << 16)
                if recsize == "big":
                    #d.float1 = NA.array([i]*32, NA.float64)
                    #d.float2 = NA.array([i**2]*32, NA.float64)
                    arr[0] = 1.1
                    d.float1 = arr
                    arr2[0] = 2.2
                    d.float2 = arr2
                    pass
                else:
                    d.float1 = NA.array([i ** 2] * 2, NA.float64)
                    #d.float1 = float(i)
                    #d.float2 = float(i)
                d.grid_i = i
                d.grid_j = 10 - i
                d.pressure = float(i * i)
                d.energy = float(d.pressure ** 4)
                table.append((d.ADCcount, d.energy, d.float1, d.float2,
                              d.grid_i, d.grid_j, d.name, d.pressure))
                # Only on float case
                # table.append((d.ADCcount, d.energy, d.float1,
                #              d.grid_i, d.grid_j, d.name, d.pressure))
        else:
            for i in range(totalrows):
                d.var1 = str(i)
                d.var2 = i
                d.var3 = 12.1e10
                table.append((d.var1, d.var2, d.var3))

        # Save this table on disk
        fileh[tablename] = table
        rowswritten += totalrows

    # Close the file
    fileh.close()
    return (rowswritten, struct.calcsize(d._v_fmt))

def readFile(filename, recsize):
    # Open the HDF5 file in read-only mode
    fileh = shelve.open(filename, "r")
    for table in ['tuple0', 'tuple1', 'tuple2']:
        if recsize == "big" or recsize == "medium":
            e = [t[2] for t in fileh[table] if t[4] < 20]
            # if there is only one float (array)
            #e = [ t[1] for t in fileh[table] if t[3] < 20 ]
        else:
            e = [t[1] for t in fileh[table] if t[1] < 20]

        print("resulting selection list ==>", e)
        print("Total selected records ==> ", len(e))

    # Close the file (eventually destroy the extended type)
    fileh.close()

# Add code to test here
if __name__ == "__main__":
    import getopt
    from time import perf_counter as clock

    usage = """usage: %s [-f] [-s recsize] [-i iterations] file
            -s use [big] record, [medium] or [small]
            -i sets the number of rows in each table\n""" % sys.argv[0]

    try:
        opts, pargs = getopt.getopt(sys.argv[1:], 's:fi:')
    except:
        sys.stderr.write(usage)
        sys.exit(0)

    # if we pass too much parameters, abort
    if len(pargs) != 1:
        sys.stderr.write(usage)
        sys.exit(0)

    # default options
    recsize = "medium"
    iterations = 100

    # Get the options
    for option in opts:
        if option[0] == '-s':
            recsize = option[1]
            if recsize not in ["big", "medium", "small"]:
                sys.stderr.write(usage)
                sys.exit(0)
        elif option[0] == '-i':
            iterations = int(option[1])

    # Catch the hdf5 file passed as the last argument
    file = pargs[0]

    t1 = clock()
    psyco.bind(createFile)
    (rowsw, rowsz) = createFile(file, iterations, recsize)
    t2 = clock()
    tapprows = t2 - t1

    t1 = clock()
    psyco.bind(readFile)
    readFile(file, recsize)
    t2 = clock()
    treadrows = t2 - t1

    print(f"Rows written: {rowsw}  Row size: {rowsz}")
    print(f"Time appending rows: {tapprows:.3f}")
    print(f"Write rows/sec: {iterations * 3 / tapprows:.0f}")
    print(f"Write KB/s : {rowsw * rowsz / (tapprows * 1024):.0f}")
    print(f"Time reading rows: {treadrows:.3f}")
    print(f"Read rows/sec:  {iterations * 3 / treadrows:.0f}")
    print(f"Read KB/s : {rowsw * rowsz / (treadrows * 1024):.0f}")
<commit_msg>shelve-bench: use normal import for the numpy module<commit_after>#!/usr/bin/env python

from tables import *
import numpy as np
import struct
import sys
import shelve
import psyco

# This class is accessible only for the examples

class Small(IsDescription):

    """Record descriptor.

    A record has several columns. They are represented here as class
    attributes, whose names are the column names and their values will
    become their types. The IsDescription class will take care the user
    will not add any new variables and that its type is correct.

    """

    var1 = StringCol(itemsize=4)
    var2 = Int32Col()
    var3 = Float64Col()

# Define a user record to characterize some kind of particles

class Medium(IsDescription):
    name = StringCol(itemsize=16)   # 16-character String
    float1 = Float64Col(shape=2, dflt=2.3)
    #float1 = Float64Col(dflt=1.3)
    #float2 = Float64Col(dflt=2.3)
    ADCcount = Int16Col()           # signed short integer
    grid_i = Int32Col()             # integer
    grid_j = Int32Col()             # integer
    pressure = Float32Col()         # float  (single-precision)
    energy = Float64Col()           # double (double-precision)

# Define a user record to characterize some kind of particles

class Big(IsDescription):
    name = StringCol(itemsize=16)   # 16-character String
    #float1 = Float64Col(shape=32, dflt=np.arange(32))
    #float2 = Float64Col(shape=32, dflt=np.arange(32))
    float1 = Float64Col(shape=32, dflt=range(32))
    float2 = Float64Col(shape=32, dflt=[2.2] * 32)
    ADCcount = Int16Col()           # signed short integer
    grid_i = Int32Col()             # integer
    grid_j = Int32Col()             # integer
    pressure = Float32Col()         # float  (single-precision)
    energy = Float64Col()           # double (double-precision)

def createFile(filename, totalrows, recsize):

    # Open a 'n'ew file
    fileh = shelve.open(filename, flag="n")

    rowswritten = 0
    # Get the record object associated with the new table
    if recsize == "big":
        d = Big()
        arr = np.array(np.arange(32), type=np.float64)
        arr2 = np.array(np.arange(32), type=np.float64)
    elif recsize == "medium":
        d = Medium()
    else:
        d = Small()
    # print d
    # sys.exit(0)
    for j in range(3):
        # Create a table
        # table = fileh.create_table(group, 'tuple'+str(j), Record(), title,
        #                          compress = 6, expectedrows = totalrows)
        # Create a Table instance
        tablename = 'tuple' + str(j)
        table = []
        # Fill the table
        if recsize == "big" or recsize == "medium":
            for i in range(totalrows):
                d.name = 'Particle: %6d' % (i)
                #d.TDCcount = i % 256
                d.ADCcount = (i * 256) % (1 << 16)
                if recsize == "big":
                    #d.float1 = np.array([i]*32, np.float64)
                    #d.float2 = np.array([i**2]*32, np.float64)
                    arr[0] = 1.1
                    d.float1 = arr
                    arr2[0] = 2.2
                    d.float2 = arr2
                    pass
                else:
                    d.float1 = np.array([i ** 2] * 2, np.float64)
                    #d.float1 = float(i)
                    #d.float2 = float(i)
                d.grid_i = i
                d.grid_j = 10 - i
                d.pressure = float(i * i)
                d.energy = float(d.pressure ** 4)
                table.append((d.ADCcount, d.energy, d.float1, d.float2,
                              d.grid_i, d.grid_j, d.name, d.pressure))
                # Only on float case
                # table.append((d.ADCcount, d.energy, d.float1,
                #              d.grid_i, d.grid_j, d.name, d.pressure))
        else:
            for i in range(totalrows):
                d.var1 = str(i)
                d.var2 = i
                d.var3 = 12.1e10
                table.append((d.var1, d.var2, d.var3))

        # Save this table on disk
        fileh[tablename] = table
        rowswritten += totalrows

    # Close the file
    fileh.close()
    return (rowswritten, struct.calcsize(d._v_fmt))

def readFile(filename, recsize):
    # Open the HDF5 file in read-only mode
    fileh = shelve.open(filename, "r")
    for table in ['tuple0', 'tuple1', 'tuple2']:
        if recsize == "big" or recsize == "medium":
            e = [t[2] for t in fileh[table] if t[4] < 20]
            # if there is only one float (array)
            #e = [ t[1] for t in fileh[table] if t[3] < 20 ]
        else:
            e = [t[1] for t in fileh[table] if t[1] < 20]

        print("resulting selection list ==>", e)
        print("Total selected records ==> ", len(e))

    # Close the file (eventually destroy the extended type)
    fileh.close()

# Add code to test here
if __name__ == "__main__":
    import getopt
    from time import perf_counter as clock

    usage = """usage: %s [-f] [-s recsize] [-i iterations] file
            -s use [big] record, [medium] or [small]
            -i sets the number of rows in each table\n""" % sys.argv[0]

    try:
        opts, pargs = getopt.getopt(sys.argv[1:], 's:fi:')
    except:
        sys.stderr.write(usage)
        sys.exit(0)

    # if we pass too much parameters, abort
    if len(pargs) != 1:
        sys.stderr.write(usage)
        sys.exit(0)

    # default options
    recsize = "medium"
    iterations = 100

    # Get the options
    for option in opts:
        if option[0] == '-s':
            recsize = option[1]
            if recsize not in ["big", "medium", "small"]:
                sys.stderr.write(usage)
                sys.exit(0)
        elif option[0] == '-i':
            iterations = int(option[1])

    # Catch the hdf5 file passed as the last argument
    file = pargs[0]

    t1 = clock()
    psyco.bind(createFile)
    (rowsw, rowsz) = createFile(file, iterations, recsize)
    t2 = clock()
    tapprows = t2 - t1

    t1 = clock()
    psyco.bind(readFile)
    readFile(file, recsize)
    t2 = clock()
    treadrows = t2 - t1

    print(f"Rows written: {rowsw}  Row size: {rowsz}")
    print(f"Time appending rows: {tapprows:.3f}")
    print(f"Write rows/sec: {iterations * 3 / tapprows:.0f}")
    print(f"Write KB/s : {rowsw * rowsz / (tapprows * 1024):.0f}")
    print(f"Time reading rows: {treadrows:.3f}")
    print(f"Read rows/sec:  {iterations * 3 / treadrows:.0f}")
    print(f"Read KB/s : {rowsw * rowsz / (treadrows * 1024):.0f}")
<|endoftext|>


<commit_before>"""
Camper+ Web Application
"""

from flask import Flask

app = Flask(__name__)
import camperapp.routes
<commit_msg>added Database<commit_after>"""
Camper+ Web Application
"""

from flask import Flask

app = Flask(__name__)
import camperapp.routes
import login.routes<|endoftext|>


<commit_before># Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import logging
import tempfile

from telemetry.internal.platform import tracing_agent
from tracing.trace_data import trace_data

from py_trace_event import trace_event
from py_trace_event import trace_time

def IsAgentEnabled():
  """Returns True if the agent is currently enabled and tracing."""
  return trace_event.trace_is_enabled()

def RecordBenchmarkMetadata(results):
  """Record benchmark metadata if tracing is enabled."""
  if IsAgentEnabled():
    trace_event.trace_add_benchmark_metadata(
        benchmark_name=results.benchmark_name,
        benchmark_description=results.benchmark_description,
        benchmark_start_time_us=results.benchmark_start_us,
        label=results.label,
        story_run_time_us=results.current_story_run.start_us,
        story_name=results.current_story.name,
        story_tags=results.current_story.GetStoryTagsList(),
        story_run_index=results.current_story_run.index,
    )
  else:
    logging.warning(
        'Telemetry tracing agent is not enabled. Discarding Telemetry info.')

def RecordIssuerClockSyncMarker(sync_id, issue_ts):
  """Record clock sync event.

  Args:
    sync_id: Unique id for sync event.
    issue_ts: timestamp before issuing clock sync to agent.
  """
  trace_event.clock_sync(sync_id, issue_ts=issue_ts)

class TelemetryTracingAgent(tracing_agent.TracingAgent):
  """Tracing agent to collect data about Telemetry (and Python) execution.

  Is implemented as a thin wrapper around py_trace_event.trace_event. Clients
  can record events using the trace_event.trace() decorator or the
  trace_event.TracedMetaClass to trace all methods of a class.

  Also works as the clock sync recorder, against which other tracing agents
  can issue clock sync events. And is responsible for recording telemetry
  metadata with information e.g. about the benchmark that produced a trace.
  """
  def __init__(self, platform_backend, config):
    super(TelemetryTracingAgent, self).__init__(platform_backend, config)
    self._trace_file = None

    # When tracing Chrome on Android, Telemetry and the browser run on
    # different devices. The trace_set_clock_snapshot() function provides
    # a clock snapshot for the perfetto trace processor to synchronize
    # host and device clocks.
    if platform_backend and platform_backend.GetOSName() == 'android':
      telemetry_ts = trace_time.Now()
      device_uptime = platform_backend.device.RunShellCommand(
          ['cat', '/proc/uptime'], single_line=True)
      boottime_ts = float(device_uptime.split(' ')[0]) * 1e6
      trace_event.trace_set_clock_snapshot(
          telemetry_ts=telemetry_ts,
          boottime_ts=boottime_ts,
      )

  @classmethod
  def IsSupported(cls, platform_backend):
    del platform_backend  # Unused.
    return trace_event.is_tracing_controllable()

  @property
  def is_tracing(self):
    return IsAgentEnabled()

  def StartAgentTracing(self, config, timeout):
    del timeout  # Unused.
    assert not self.is_tracing, 'Telemetry tracing already running'

    # Keep Telemetry trace format in accordance with Chrome trace format.
    if config and config.chrome_trace_config.trace_format == 'proto':
      trace_format = trace_event.PROTOBUF
      suffix = '.pb'
    else:
      trace_format = trace_event.JSON_WITH_METADATA
      suffix = '.json'

    # Create a temporary file and pass the opened file-like object to
    # trace_event.trace_enable(); the file will be closed on trace_disable(),
    # and later passed to a trace data builder in CollectAgentTraceData().
    self._trace_file = tempfile.NamedTemporaryFile(delete=False, suffix=suffix)
    trace_event.trace_enable(self._trace_file, format=trace_format)
    assert self.is_tracing, 'Failed to start Telemetry tracing'
    return True

  def StopAgentTracing(self):
    assert self.is_tracing, 'Telemetry tracing is not running'
    trace_event.trace_disable()
    assert not self.is_tracing, 'Failed to stop Telemetry tracing'

  def CollectAgentTraceData(self, trace_data_builder, timeout=None):
    assert not self.is_tracing, 'Must stop tracing before collection'
    assert self._trace_file.closed, 'Trace file was not properly closed'
    # Ownership of the file, and responsibility to manage its lifetime, is now
    # transferred over to the trace data builder.
    trace_data_builder.AddTraceFileFor(
        trace_data.TELEMETRY_PART, self._trace_file.name)
    self._trace_file = None
<commit_msg>Reland "[Telemetry] Only use Perfetto sync when all agents produce proto traces"<commit_after># Copyright 2019 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import logging
import tempfile

from telemetry.internal.platform import tracing_agent
from tracing.trace_data import trace_data

from py_trace_event import trace_event
from py_trace_event import trace_time

def IsAgentEnabled():
  """Returns True if the agent is currently enabled and tracing."""
  return trace_event.trace_is_enabled()

def RecordBenchmarkMetadata(results):
  """Record benchmark metadata if tracing is enabled."""
  if IsAgentEnabled():
    trace_event.trace_add_benchmark_metadata(
        benchmark_name=results.benchmark_name,
        benchmark_description=results.benchmark_description,
        benchmark_start_time_us=results.benchmark_start_us,
        label=results.label,
        story_run_time_us=results.current_story_run.start_us,
        story_name=results.current_story.name,
        story_tags=results.current_story.GetStoryTagsList(),
        story_run_index=results.current_story_run.index,
    )
  else:
    logging.warning(
        'Telemetry tracing agent is not enabled. Discarding Telemetry info.')

def RecordIssuerClockSyncMarker(sync_id, issue_ts):
  """Record clock sync event.

  Args:
    sync_id: Unique id for sync event.
    issue_ts: timestamp before issuing clock sync to agent.
  """
  trace_event.clock_sync(sync_id, issue_ts=issue_ts)

def _UsePerfettoSync(platform_backend, config):
  is_android = (platform_backend is not None
                and platform_backend.GetOSName() == 'android')
  if config is not None and config.enable_experimental_system_tracing:
    all_agents_support_proto = True
  else:
    all_agents_support_proto = config is not None and not (
        config.enable_atrace_trace or
        config.enable_cpu_trace or
        config.enable_platform_display_trace)
  return is_android and all_agents_support_proto

class TelemetryTracingAgent(tracing_agent.TracingAgent):
  """Tracing agent to collect data about Telemetry (and Python) execution.

  Is implemented as a thin wrapper around py_trace_event.trace_event. Clients
  can record events using the trace_event.trace() decorator or the
  trace_event.TracedMetaClass to trace all methods of a class.

  Also works as the clock sync recorder, against which other tracing agents
  can issue clock sync events. And is responsible for recording telemetry
  metadata with information e.g. about the benchmark that produced a trace.
  """
  def __init__(self, platform_backend, config):
    super(TelemetryTracingAgent, self).__init__(platform_backend, config)
    self._trace_file = None

    # When tracing Chrome on Android, Telemetry and the browser run on
    # different devices. The trace_set_clock_snapshot() function provides
    # a clock snapshot for the perfetto trace processor to synchronize
    # host and device clocks.
    if _UsePerfettoSync(platform_backend, config):
      telemetry_ts = trace_time.Now()
      device_uptime = platform_backend.device.RunShellCommand(
          ['cat', '/proc/uptime'], single_line=True)
      boottime_ts = float(device_uptime.split(' ')[0]) * 1e6
      trace_event.trace_set_clock_snapshot(
          telemetry_ts=telemetry_ts,
          boottime_ts=boottime_ts,
      )

  @classmethod
  def IsSupported(cls, platform_backend):
    del platform_backend  # Unused.
    return trace_event.is_tracing_controllable()

  @property
  def is_tracing(self):
    return IsAgentEnabled()

  def StartAgentTracing(self, config, timeout):
    del timeout  # Unused.
    assert not self.is_tracing, 'Telemetry tracing already running'

    # Keep Telemetry trace format in accordance with Chrome trace format.
    if config and config.chrome_trace_config.trace_format == 'proto':
      trace_format = trace_event.PROTOBUF
      suffix = '.pb'
    else:
      trace_format = trace_event.JSON_WITH_METADATA
      suffix = '.json'

    # Create a temporary file and pass the opened file-like object to
    # trace_event.trace_enable(); the file will be closed on trace_disable(),
    # and later passed to a trace data builder in CollectAgentTraceData().
    self._trace_file = tempfile.NamedTemporaryFile(delete=False, suffix=suffix)
    trace_event.trace_enable(self._trace_file, format=trace_format)
    assert self.is_tracing, 'Failed to start Telemetry tracing'
    return True

  def StopAgentTracing(self):
    assert self.is_tracing, 'Telemetry tracing is not running'
    trace_event.trace_disable()
    assert not self.is_tracing, 'Failed to stop Telemetry tracing'

  def CollectAgentTraceData(self, trace_data_builder, timeout=None):
    assert not self.is_tracing, 'Must stop tracing before collection'
    assert self._trace_file.closed, 'Trace file was not properly closed'
    # Ownership of the file, and responsibility to manage its lifetime, is now
    # transferred over to the trace data builder.
    trace_data_builder.AddTraceFileFor(
        trace_data.TELEMETRY_PART, self._trace_file.name)
    self._trace_file = None
<|endoftext|>


<commit_before># This file is part of BenchExec, a framework for reliable benchmarking:
# https://github.com/sosy-lab/benchexec
#
# SPDX-FileCopyrightText: 2007-2020 Dirk Beyer <https://www.sosy-lab.org>
#
# SPDX-License-Identifier: Apache-2.0

"""Main package of BenchExec.

The following modules are the public entry points:
- benchexec: for executing benchmark suites
- runexec: for executing single runs
- check_cgroups: for checking the availability of cgroups
- test_tool_info: for testing a tool-info module

Naming conventions used within BenchExec:

TOOL: a (verification) tool that should be executed
EXECUTABLE: the executable file that should be called for running a TOOL
INPUTFILE: one input file for the TOOL
RUN: one execution of a TOOL on one INPUTFILE
RUNSET: a set of RUNs of one TOOL with at most one RUN per INPUTFILE
RUNDEFINITION: a template for the creation of a RUNSET with RUNS from one or more INPUTFILESETs
BENCHMARK: a list of RUNDEFINITIONs and INPUTFILESETs for one TOOL
OPTION: a user-specified option to add to the command-line of the TOOL when it its run
CONFIG: the configuration of this script consisting of the command-line arguments given by the user
EXECUTOR: a module for executing a BENCHMARK

"run" always denotes a job to do and is never used as a verb.
"execute" is only used as a verb (this is what is done with a run).
A benchmark or a run set can also be executed, which means to execute all contained runs.

Variables ending with "file" contain filenames.
Variables ending with "tag" contain references to XML tag objects created by the XML parser.
"""

__version__ = "3.2"

class BenchExecException(Exception):
    pass
<commit_msg>Prepare version number for next development cycle.<commit_after># This file is part of BenchExec, a framework for reliable benchmarking:
# https://github.com/sosy-lab/benchexec
#
# SPDX-FileCopyrightText: 2007-2020 Dirk Beyer <https://www.sosy-lab.org>
#
# SPDX-License-Identifier: Apache-2.0

"""Main package of BenchExec.

The following modules are the public entry points:
- benchexec: for executing benchmark suites
- runexec: for executing single runs
- check_cgroups: for checking the availability of cgroups
- test_tool_info: for testing a tool-info module

Naming conventions used within BenchExec:

TOOL: a (verification) tool that should be executed
EXECUTABLE: the executable file that should be called for running a TOOL
INPUTFILE: one input file for the TOOL
RUN: one execution of a TOOL on one INPUTFILE
RUNSET: a set of RUNs of one TOOL with at most one RUN per INPUTFILE
RUNDEFINITION: a template for the creation of a RUNSET with RUNS from one or more INPUTFILESETs
BENCHMARK: a list of RUNDEFINITIONs and INPUTFILESETs for one TOOL
OPTION: a user-specified option to add to the command-line of the TOOL when it its run
CONFIG: the configuration of this script consisting of the command-line arguments given by the user
EXECUTOR: a module for executing a BENCHMARK

"run" always denotes a job to do and is never used as a verb.
"execute" is only used as a verb (this is what is done with a run).
A benchmark or a run set can also be executed, which means to execute all contained runs.

Variables ending with "file" contain filenames.
Variables ending with "tag" contain references to XML tag objects created by the XML parser.
"""

__version__ = "3.3-dev"

class BenchExecException(Exception):
    pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import six
import copy
import logging

from .field import Field

class Model(object):
    """Model base class. Subclasses should define `Field` descriptors, which
    are collected in `_fields`. Note: to avoid circular definitions, the
    `ModelMeta` metaclass is not applied until after `Model` has been defined.

    """
    def __init__(self, **kwargs):
        self.unserialize(kwargs)

    def serialize_key(self, key):
        return key

    def unserialize_key(self, key):
        return key

    def serialize(self):
        return {
            self.serialize_key(key): value.serialize(self)
            for key, value in six.iteritems(self._fields)
        }

    def unserialize(self, kwargs):
        for key, value in six.iteritems(kwargs):
            key = self.unserialize_key(key)
            if key not in self._fields:
                logging.warning('Key {0} not in model schema'.format(key))
            else:
                self._fields[key].__set__(self, value, safe=True)
        self.check_complete()

    def check_complete(self):
        missing = [
            key for key, value in six.iteritems(self._fields)
            if value.required
            and value.is_null(value.__get__(self, self.__class__))
        ]
        if missing:
            raise ValueError('Missing values on fields {0}'.format(
                ', '.join(missing)
            ))

def copy_fields(bases):
    """Deep-copy and collect `Field` descriptors from base classes.

    :param list bases: Base classes
    :returns: Dictionary of `Field` descriptors

    """
    fields = {}
    for base in bases[::-1]:
        if issubclass(base, Model):
            fields.update({
                key: copy.deepcopy(value)
                for key, value in six.iteritems(base._fields)
            })
    return fields

class ModelMeta(type):
    """Model metaclass. Collects field descriptors and inherits fields from
    parent.

    """
    def __init__(cls, name, bases, dct):
        cls._fields = copy_fields(bases)
        cls._fields.update({
            key: value
            for key, value in six.iteritems(dct)
            if isinstance(value, Field)
        })
        super(ModelMeta, cls).__init__(name, bases, dct)

# Apply metaclass to `Model`
Model = six.add_metaclass(ModelMeta)(Model)
<commit_msg>Use named logger.<commit_after># -*- coding: utf-8 -*-

import six
import copy
import logging

from .field import Field

logger = logging.getLogger(__name__)

class Model(object):
    """Model base class. Subclasses should define `Field` descriptors, which
    are collected in `_fields`. Note: to avoid circular definitions, the
    `ModelMeta` metaclass is not applied until after `Model` has been defined.

    """
    def __init__(self, **kwargs):
        self.unserialize(kwargs)

    def serialize_key(self, key):
        return key

    def unserialize_key(self, key):
        return key

    def serialize(self):
        return {
            self.serialize_key(key): value.serialize(self)
            for key, value in six.iteritems(self._fields)
        }

    def unserialize(self, kwargs):
        for key, value in six.iteritems(kwargs):
            key = self.unserialize_key(key)
            if key not in self._fields:
                logger.warn('Key {0} not in model schema'.format(key))
            else:
                self._fields[key].__set__(self, value, safe=True)
        self.check_complete()

    def check_complete(self):
        missing = [
            key for key, value in six.iteritems(self._fields)
            if value.required
            and value.is_null(value.__get__(self, self.__class__))
        ]
        if missing:
            raise ValueError('Missing values on fields {0}'.format(
                ', '.join(missing)
            ))

def copy_fields(bases):
    """Deep-copy and collect `Field` descriptors from base classes.

    :param list bases: Base classes
    :returns: Dictionary of `Field` descriptors

    """
    fields = {}
    for base in bases[::-1]:
        if issubclass(base, Model):
            fields.update({
                key: copy.deepcopy(value)
                for key, value in six.iteritems(base._fields)
            })
    return fields

class ModelMeta(type):
    """Model metaclass. Collects field descriptors and inherits fields from
    parent.

    """
    def __init__(cls, name, bases, dct):
        cls._fields = copy_fields(bases)
        cls._fields.update({
            key: value
            for key, value in six.iteritems(dct)
            if isinstance(value, Field)
        })
        super(ModelMeta, cls).__init__(name, bases, dct)

# Apply metaclass to `Model`
Model = six.add_metaclass(ModelMeta)(Model)
<|endoftext|>


<commit_before># Copyright (c) 2016 P4FPGA Project
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#

import config
import os
import pprint

def CamelCase(name):
    output = ''.join(x for x in name.title() if x.isalnum())
    return output

def camelCase(name):
    output = ''.join(x for x in name.title() if x.isalnum())
    return output[0].lower() + output[1:]

def GetHeaderTypeWidth(header_type):
    assert type(header_type) == str
    for h in config.jsondata["header_types"]:
        if h["name"] == header_type:
            fields = h["fields"]
            width = sum([x for _, x in fields])
            return width
    return None

def GetHeaderWidth(header):
    assert type(header) == str
    #print 'htow', header
    for h in config.jsondata["headers"]:
        if h["name"] == header:
            hty = h["header_type"]
            return GetHeaderTypeWidth(hty)
    return None

def GetFieldWidth(field):
    #assert type(field) is list
    hty = None
    fields = None
    for h in config.jsondata["headers"]:
        if h["name"] == field[0]:
            hty = h["header_type"]

    for h in config.jsondata["header_types"]:
        if h["name"] == hty:
            fields = h["fields"]

    for f, width in fields:
        if f == field[1]:
            return width
    return None

def GetHeaderType(header):
    assert type(header) == str
    for h in config.jsondata["headers"]:
        if h["name"] == header:
            return h["header_type"]
    return None

def state_name_to_state (state_name):
    for s in config.jsondata['parsers'][0]['parse_states']:
        if s['name'] == state_name:
            return s
    return None

def state_to_header (state_name):
    state = state_name_to_state(state_name)
    headers = []
    header_stacks = []
    stack = False
    for op in state["parser_ops"]:
        if op["op"] == "extract":
            parameters = op['parameters'][0]
            if parameters['type'] == "regular":
                value = parameters["value"]
                headers.append(value)
            elif parameters['type'] == 'stack':
                value = parameters['value']
                headers.append("%s[%d]" % (value, 0))
    return headers

def build_expression(json_data, sb=[], metadata=[]):
    if not json_data:
        return
    json_type = json_data["type"]
    json_value = json_data["value"]
    if (json_type == "expression"):
        op = json_value["op"]
        json_left = json_value["left"]
        json_right = json_value["right"]

        sb.append("(")
        if (op == "?"):
            json_cond = json_data["cond"]
            build_expression(value["left"], sb, metadata)
            sb.append(op)
            build_expression(value["right"], sb, metadata)
            sb.append(")")
        else:
            if ((op == "+") or op == "-") and json_left is None:
                print "expr push back load const"
            else:
                build_expression(json_left, sb, metadata)
            sb.append(op)
            build_expression(json_right, sb, metadata)
            sb.append(")")
    elif (json_type == "header"):
        if type(json_value) == list:
            sb.append("$".join(json_value))
        else:
            sb.append(json_value)
        metadata.append(json_value)
    elif (json_type == "field"):
        if type(json_value) == list:
            sb.append("$".join(json_value))
        else:
            sb.append(json_value)
        metadata.append(json_value)
    elif (json_type == "bool"):
        sb.append(json_value)
    elif (json_type == "hexstr"):
        sb.append(json_value)
    elif (json_type == "local"):
        sb.append(json_value)
    elif (json_type == "register"):
        sb.append(json_value)
    else:
        assert "Error: unimplemented expression type", json_type

def state_to_expression (state_name):
    state = state_name_to_state(state_name)
    # HACK: dealing with BMV2 json format
    for op in state['parser_ops']:
        src = []
        dst = []
        dst_hdr = []
        src_hdr = []
        if op['op'] == 'set':
            exp0 = op['parameters'][0]
            build_expression(exp0, dst, dst_hdr)
            exp1 = op['parameters'][1]
            if exp1['type'] == 'expression':
                if exp1['value']:
                    build_expression(exp1['value'], src, [])
                    return 'expression', dst, src
            else:
                build_expression(exp1, [], src)
                return 'field', dst, src[0]
    return None, None, None

def createDirAndOpen(f, m):
    (d, name) = os.path.split(f)
    if not os.path.exists(d):
        os.makedirs(d)
    return open(f, m)

<commit_msg>state_name_to_state -> GetState<commit_after># Copyright (c) 2016 P4FPGA Project
#
# Permission is hereby granted, free of charge, to any person obtaining a
# copy of this software and associated documentation files (the "Software"),
# to deal in the Software without restriction, including without limitation
# the rights to use, copy, modify, merge, publish, distribute, sublicense,
# and/or sell copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
# DEALINGS IN THE SOFTWARE.
#

import config
import os
import pprint

def CamelCase(name):
    output = ''.join(x for x in name.title() if x.isalnum())
    return output

def camelCase(name):
    output = ''.join(x for x in name.title() if x.isalnum())
    return output[0].lower() + output[1:]

def GetHeaderTypeWidth(header_type):
    assert type(header_type) == str
    for h in config.jsondata["header_types"]:
        if h["name"] == header_type:
            fields = h["fields"]
            width = sum([x for _, x in fields])
            return width
    return None

def GetHeaderWidth(header):
    assert type(header) == str
    #print 'htow', header
    for h in config.jsondata["headers"]:
        if h["name"] == header:
            hty = h["header_type"]
            return GetHeaderTypeWidth(hty)
    return None

def GetFieldWidth(field):
    #assert type(field) is list
    hty = None
    fields = None
    for h in config.jsondata["headers"]:
        if h["name"] == field[0]:
            hty = h["header_type"]

    for h in config.jsondata["header_types"]:
        if h["name"] == hty:
            fields = h["fields"]

    for f, width in fields:
        if f == field[1]:
            return width
    return None

def GetHeaderType(header):
    assert type(header) == str
    for h in config.jsondata["headers"]:
        if h["name"] == header:
            return h["header_type"]
    return None

def GetState (state_name):
    for s in config.jsondata['parsers'][0]['parse_states']:
        if s['name'] == state_name:
            return s
    return None

def state_to_header (state_name):
    state = GetState(state_name)
    headers = []
    header_stacks = []
    stack = False
    for op in state["parser_ops"]:
        if op["op"] == "extract":
            parameters = op['parameters'][0]
            if parameters['type'] == "regular":
                value = parameters["value"]
                headers.append(value)
            elif parameters['type'] == 'stack':
                value = parameters['value']
                headers.append("%s[%d]" % (value, 0))
    return headers

def build_expression(json_data, sb=[], metadata=[]):
    if not json_data:
        return
    json_type = json_data["type"]
    json_value = json_data["value"]
    if (json_type == "expression"):
        op = json_value["op"]
        json_left = json_value["left"]
        json_right = json_value["right"]

        sb.append("(")
        if (op == "?"):
            json_cond = json_data["cond"]
            build_expression(value["left"], sb, metadata)
            sb.append(op)
            build_expression(value["right"], sb, metadata)
            sb.append(")")
        else:
            if ((op == "+") or op == "-") and json_left is None:
                print "expr push back load const"
            else:
                build_expression(json_left, sb, metadata)
            sb.append(op)
            build_expression(json_right, sb, metadata)
            sb.append(")")
    elif (json_type == "header"):
        if type(json_value) == list:
            sb.append("$".join(json_value))
        else:
            sb.append(json_value)
        metadata.append(json_value)
    elif (json_type == "field"):
        if type(json_value) == list:
            sb.append("$".join(json_value))
        else:
            sb.append(json_value)
        metadata.append(json_value)
    elif (json_type == "bool"):
        sb.append(json_value)
    elif (json_type == "hexstr"):
        sb.append(json_value)
    elif (json_type == "local"):
        sb.append(json_value)
    elif (json_type == "register"):
        sb.append(json_value)
    else:
        assert "Error: unimplemented expression type", json_type

def state_to_expression (state_name):
    state = GetState(state_name)
    # HACK: dealing with BMV2 json format
    for op in state['parser_ops']:
        src = []
        dst = []
        dst_hdr = []
        src_hdr = []
        if op['op'] == 'set':
            exp0 = op['parameters'][0]
            build_expression(exp0, dst, dst_hdr)
            exp1 = op['parameters'][1]
            if exp1['type'] == 'expression':
                if exp1['value']:
                    build_expression(exp1['value'], src, [])
                    return 'expression', dst, src
            else:
                build_expression(exp1, [], src)
                return 'field', dst, src[0]
    return None, None, None

def createDirAndOpen(f, m):
    (d, name) = os.path.split(f)
    if not os.path.exists(d):
        os.makedirs(d)
    return open(f, m)

<|endoftext|>


<commit_before># the data import manager
# gets an uploaded file or retrieves a file from a URL
# Uses the parser manager to parse the file
# indexes the records found in the file by upserting via the DAO
import urllib2
from datetime import datetime
import re
from cStringIO import StringIO
import unicodedata

from bibserver.parser import Parser
import bibserver.dao
import bibserver.util as util

class Importer(object):
    def __init__(self, owner):
        self.owner = owner

    def upload(self, fileobj, format_, collection=None):
        '''Import a collection into the database.

        :param fileobj: a fileobj pointing to file from which to import
        collection records (and possibly collection metadata)
        :param format_: format of the fileobj (e.g. bibtex)
        :param collection: collection dict for use when creating collection. If
        undefined collection must be extractable from the fileobj.

        :return: same as `index` method.
        '''
        parser = Parser()
        record_dicts = parser.parse(fileobj, format=format_)
        #collection_from_parser = None
        #if collection_from_parser:
        #    collection = collection_from_parser
        # TODO: check authz for write to this collection
        return self.index(collection, record_dicts)

    def upload_from_web(self, request):
        '''
        :param request_data: Flask request.values attribute.
        '''
        format = 'bibtex'
        source = ''
        if request.values.get("source"):
            source = urllib2.unquote(request.values.get("source", ''))
            fileobj = urllib2.urlopen(source)
            format = self.findformat(source)
        elif request.files.get('upfile'):
            fileobj = request.files.get('upfile')
            format = self.findformat(fileobj.filename)
        elif request.values.get('data'):
            fileobj = StringIO(request.values['data'])
        if request.values.get('format'):
            format = request.values.get('format')

        if not 'collection' in request.values:
            raise ValueError('You must provide a collection label')
        collection = {
            'label': request.values['collection'],
            'source': source,
            'format': format
            }
        collection, records = self.upload(fileobj, format, collection)
        return (collection, records)

    def findformat(self,filename):
        if filename.endswith(".json"): return "json"
        if filename.endswith(".bibjson"): return "bibjson"
        if filename.endswith(".bibtex"): return "bibtex"
        if filename.endswith(".bib"): return "bibtex"
        if filename.endswith(".csv"): return "csv"
        return "bibtex"

    def bulk_upload(self, colls_list):
        '''upload a list of collections from provided file locations.

        :param colls_list: a list of dictionaries with 3 keys::

            {
                # source = url source for data
                # upfile = local file path for data
                # data = raw data
                source | upfile | data: ...,
                format: {the-format-of-the-data-e.g.-bibtex},
                collection: {label for the collection}
            }
        '''
        for coll in colls_list["collections"]:
            if "upfile" in coll:
                fileobj = coll["upfile"]
            elif "data" in coll:
                fileobj = StringIO(coll['data'])
            elif "source" in coll:
                fileobj = urllib2.urlopen(coll["source"])
            format_ = coll['format']
            collection_dict = {
                'label': coll['collection']
                }
            self.upload(fileobj, format_, collection_dict)
        return True

    def index(self, collection_dict, record_dicts):
        '''Add this collection and its records to the database index.

        :return: (collection, records) tuple of collection and associated
        record objects.
        '''
        collection = bibserver.dao.Collection(**collection_dict)
        timestamp = datetime.now().isoformat()
        collection['created'] = timestamp
        assert 'label' in collection, 'Collection must have a label'
        if not 'slug' in collection:
            collection['slug'] = util.slugify(collection['label'])
        collection['owner'] = self.owner.id
        # check if there is an existing collection for this user with same
        # source, or if no source an object with same slug, and if so use that instead
        for coll in self.owner.collections:
            if 'source' in collection:
                if coll.get('source','') == collection['source']:
                    collection['id'] = coll['id']
                    break
            else:
                if coll['slug'] == collection['slug']:
                    collection['id'] = coll['id']
                    break
        collection['total_records'] = len(record_dicts)
        collection['modified'] = timestamp
        collection.save()
        # delete any old versions of the records
        # TODO: should we merge (ie. do upsert rather than delete all existing
        # ones)
        bibserver.dao.Record.delete_by_query('collection.exact:"' +
                collection['slug'] + '"')
        for rec in record_dicts:
            rec['collection'] = collection["slug"]
        records = bibserver.dao.Record.bulk_upsert(record_dicts)
        return collection, records

    # parse potential people out of a record
    # check if they have a person record in bibsoup
    # if not create one
    # append person IDs to a person attribute of every record
    def parse_people(self,record):
        if "person" not in record:
            record["person"] = []
        if "author" in record:
            record["person"].extend(self.do_people(record["author"]))
        if "advisor" in record:
            record["person"].extend(self.do_people(record["advisor"]))
        if "editor" in record:
            record["person"].extend(self.do_people(record["editor"]))
        return record

    def do_people(self,people):
        persons = []
        if isinstance(people,str):
            persons = self.do_person(people)
        if isinstance(people,list):
            for person in people:
                persons.append(self.do_person(person))
        return persons

    def do_person(self,person_string):
        try:
            results = bibserver.dao.Record.query(q='type.exact:"person" AND alias.exact:"' + person_string + '"')
            if results["hits"]["total"] != 0:
                return results["hits"]["hits"][0]["_source"]["person"]

            looseresults = bibserver.dao.Record.query(q='type.exact:"person" AND "*' + person_string + '*"',flt=True,fields=["person"])
            if looseresults["hits"]["total"] != 0:
                tid = looseresults["hits"]["hits"][0]["_id"]
                data = bibserver.dao.Record.get(tid)
                if "alias" in data:
                    if person_string not in data["alias"]:
                        data["alias"].append(person_string)
                bibserver.dao.Record.upsert(data)

                return data["person"]

            ident = person_string.replace(" ","").replace(",","").replace(".","")
            data = {"person":ident,"type":"person","alias":[person_string]}
            bibserver.dao.Record.upsert(data)
            return ident

        except:
            return []

<commit_msg>changing the importer to try to properly handle POST upload data strings<commit_after># the data import manager
# gets an uploaded file or retrieves a file from a URL
# Uses the parser manager to parse the file
# indexes the records found in the file by upserting via the DAO
import urllib2
from datetime import datetime
import re
from cStringIO import StringIO
import unicodedata

from bibserver.parser import Parser
import bibserver.dao
import bibserver.util as util

class Importer(object):
    def __init__(self, owner):
        self.owner = owner

    def upload(self, fileobj, format_, collection=None):
        '''Import a collection into the database.

        :param fileobj: a fileobj pointing to file from which to import
        collection records (and possibly collection metadata)
        :param format_: format of the fileobj (e.g. bibtex)
        :param collection: collection dict for use when creating collection. If
        undefined collection must be extractable from the fileobj.

        :return: same as `index` method.
        '''
        parser = Parser()
        record_dicts = parser.parse(fileobj, format=format_)
        #collection_from_parser = None
        #if collection_from_parser:
        #    collection = collection_from_parser
        # TODO: check authz for write to this collection
        return self.index(collection, record_dicts)

    def upload_from_web(self, request):
        '''
        :param request_data: Flask request.values attribute.
        '''
        format = 'bibtex'
        source = ''
        if request.values.get("source"):
            source = urllib2.unquote(request.values.get("source", ''))
            fileobj = urllib2.urlopen(source)
            format = self.findformat(source)
        elif request.files.get('upfile'):
            fileobj = request.files.get('upfile')
            format = self.findformat(fileobj.filename)
        elif request.json:
            # see flask repo for further info
            # https://github.com/mitsuhiko/flask/issues/110
            fileobj = StringIO(request.json)
        elif request.data:
            fileobj = StringIO(request.data)

        if request.values.get('format'):
            format = request.values.get('format')

        if not 'collection' in request.values:
            raise ValueError('You must provide a collection label')
        collection = {
            'label': request.values['collection'],
            'source': source,
            'format': format
            }
        collection, records = self.upload(fileobj, format, collection)
        return (collection, records)

    def findformat(self,filename):
        if filename.endswith(".json"): return "json"
        if filename.endswith(".bibjson"): return "bibjson"
        if filename.endswith(".bibtex"): return "bibtex"
        if filename.endswith(".bib"): return "bibtex"
        if filename.endswith(".csv"): return "csv"
        return "bibtex"

    def bulk_upload(self, colls_list):
        '''upload a list of collections from provided file locations.

        :param colls_list: a list of dictionaries with 3 keys::

            {
                # source = url source for data
                # upfile = local file path for data
                # data = raw data
                source | upfile | data: ...,
                format: {the-format-of-the-data-e.g.-bibtex},
                collection: {label for the collection}
            }
        '''
        for coll in colls_list["collections"]:
            if "upfile" in coll:
                fileobj = coll["upfile"]
            elif "data" in coll:
                fileobj = StringIO(coll['data'])
            elif "source" in coll:
                fileobj = urllib2.urlopen(coll["source"])
            format_ = coll['format']
            collection_dict = {
                'label': coll['collection']
                }
            self.upload(fileobj, format_, collection_dict)
        return True

    def index(self, collection_dict, record_dicts):
        '''Add this collection and its records to the database index.

        :return: (collection, records) tuple of collection and associated
        record objects.
        '''
        collection = bibserver.dao.Collection(**collection_dict)
        timestamp = datetime.now().isoformat()
        collection['created'] = timestamp
        assert 'label' in collection, 'Collection must have a label'
        if not 'slug' in collection:
            collection['slug'] = util.slugify(collection['label'])
        collection['owner'] = self.owner.id
        # check if there is an existing collection for this user with same
        # source, or if no source an object with same slug, and if so use that instead
        for coll in self.owner.collections:
            if 'source' in collection:
                if coll.get('source','') == collection['source']:
                    collection['id'] = coll['id']
                    break
            else:
                if coll['slug'] == collection['slug']:
                    collection['id'] = coll['id']
                    break
        collection['total_records'] = len(record_dicts)
        collection['modified'] = timestamp
        collection.save()
        # delete any old versions of the records
        # TODO: should we merge (ie. do upsert rather than delete all existing
        # ones)
        bibserver.dao.Record.delete_by_query('collection.exact:"' +
                collection['slug'] + '"')
        for rec in record_dicts:
            rec['collection'] = collection["slug"]
        records = bibserver.dao.Record.bulk_upsert(record_dicts)
        return collection, records

    # parse potential people out of a record
    # check if they have a person record in bibsoup
    # if not create one
    # append person IDs to a person attribute of every record
    def parse_people(self,record):
        if "person" not in record:
            record["person"] = []
        if "author" in record:
            record["person"].extend(self.do_people(record["author"]))
        if "advisor" in record:
            record["person"].extend(self.do_people(record["advisor"]))
        if "editor" in record:
            record["person"].extend(self.do_people(record["editor"]))
        return record

    def do_people(self,people):
        persons = []
        if isinstance(people,str):
            persons = self.do_person(people)
        if isinstance(people,list):
            for person in people:
                persons.append(self.do_person(person))
        return persons

    def do_person(self,person_string):
        try:
            results = bibserver.dao.Record.query(q='type.exact:"person" AND alias.exact:"' + person_string + '"')
            if results["hits"]["total"] != 0:
                return results["hits"]["hits"][0]["_source"]["person"]

            looseresults = bibserver.dao.Record.query(q='type.exact:"person" AND "*' + person_string + '*"',flt=True,fields=["person"])
            if looseresults["hits"]["total"] != 0:
                tid = looseresults["hits"]["hits"][0]["_id"]
                data = bibserver.dao.Record.get(tid)
                if "alias" in data:
                    if person_string not in data["alias"]:
                        data["alias"].append(person_string)
                bibserver.dao.Record.upsert(data)

                return data["person"]

            ident = person_string.replace(" ","").replace(",","").replace(".","")
            data = {"person":ident,"type":"person","alias":[person_string]}
            bibserver.dao.Record.upsert(data)
            return ident

        except:
            return []

<|endoftext|>


<commit_before>"""Utility functions to check attributes returned in API responses and read from the AWS S3."""
import datetime
import re

def check_attribute_presence(node, attribute_name):
    """Check the attribute presence in the given dictionary or list.

    To be used to check the deserialized JSON data etc.
    """
    found_attributes = node if type(node) is list else node.keys()
    assert attribute_name in node, \
        "'%s' attribute is expected in the node, " \
        "found: %s attributes " % (attribute_name, ", ".join(found_attributes))

def check_attributes_presence(node, attribute_names):
    """Check the presence of all attributes in the dictionary or in the list.

    To be used to check the deserialized JSON data etc.
    """
    for attribute_name in attribute_names:
        found_attributes = node if type(node) is list else node.keys()
        assert attribute_name in node, \
            "'%s' attribute is expected in the node, " \
            "found: %s attributes " % (attribute_name, ", ".join(found_attributes))

def check_and_get_attribute(node, attribute_name):
    """Check the attribute presence and if the attribute is found, return its value."""
    check_attribute_presence(node, attribute_name)
    return node[attribute_name]

def check_uuid(uuid):
    """Check if the string contains a proper UUID.

    Supported format: 71769af6-0a39-4242-94be-1f84f04c8a56
    """
    regex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z',
                       re.I)
    match = regex.match(uuid)
    return bool(match)

def check_timestamp(timestamp):
    """Check if the string contains proper timestamp value.

    The following four formats are supported:
    2017-07-19 13:05:25.041688
    2017-07-17T09:05:29.101780
    2017-07-19 13:05:25
    2017-07-17T09:05:29
    """
    assert timestamp is not None
    assert isinstance(timestamp, str)

    # some attributes contains timestamp without the millisecond part
    # so we need to take care of it
    if len(timestamp) == len("YYYY-mm-dd HH:MM:SS") and '.' not in timestamp:
        timestamp += '.0'

    assert len(timestamp) >= len("YYYY-mm-dd HH:MM:SS.")

    # we have to support the following formats:
    #    2017-07-19 13:05:25.041688
    #    2017-07-17T09:05:29.101780
    # -> it is needed to distinguish the 'T' separator
    #
    # (please see https://www.tutorialspoint.com/python/time_strptime.htm for
    #  an explanation how timeformat should look like)

    timeformat = "%Y-%m-%d %H:%M:%S.%f"
    if timestamp[10] == "T":
        timeformat = "%Y-%m-%dT%H:%M:%S.%f"

    # just try to parse the string to check whether
    # the ValueError exception is raised or not
    datetime.datetime.strptime(timestamp, timeformat)

def check_job_token_attributes(token):
    """Check that the given JOB token contains all required attributes."""
    attribs = ["limit", "remaining", "reset"]
    for attr in attribs:
        assert attr in token
        assert int(token[attr]) >= 0

def check_status_attribute(data):
    """Check the value of the status attribute, that should contain just two allowed values."""
    status = check_and_get_attribute(data, "status")
    assert status in ["success", "error"]

def check_summary_attribute(data):
    """Check the summary attribute that can be found all generated metadata."""
    summary = check_and_get_attribute(data, "summary")
    assert type(summary) is list or type(summary) is dict

def release_string(ecosystem, package, version=None):
    """Construct a string with ecosystem:package or ecosystem:package:version tuple."""
    return "{e}:{p}:{v}".format(e=ecosystem, p=package, v=version)

def check_release_attribute(data, ecosystem, package, version=None):
    """Check the content of _release attribute.

    Check that the attribute _release contains proper release string for given ecosystem
    and package.
    """
    check_attribute_presence(data, "_release")
    assert data["_release"] == release_string(ecosystem, package, version)

def check_schema_attribute(data, expected_schema_name, expected_schema_version):
    """Check the content of the schema attribute.

    This attribute should contains dictionary with name and version that are checked as well.
    """
    # read the toplevel attribute 'schema'
    schema = check_and_get_attribute(data, "schema")

    # read attributes from the 'schema' node
    name = check_and_get_attribute(schema, "name")
    version = check_and_get_attribute(schema, "version")

    # check the schema name
    assert name == expected_schema_name, "Schema name '{n1}' is different from " \
        "expected name '{n2}'".format(n1=name, n2=expected_schema_name)

    # check the schema version (ATM we are able to check just one fixed version)
    assert version == expected_schema_version, "Schema version {v1} is different from expected " \
        "version {v2}".format(v1=version, v2=expected_schema_version)

def check_audit_metadata(data):
    """Check the metadata stored in the _audit attribute.

    Check if all common attributes can be found in the audit node
    in the component or package metadata.
    """
    check_attribute_presence(data, "_audit")
    audit = data["_audit"]

    check_attribute_presence(audit, "version")
    assert audit["version"] == "v1"

    check_attribute_presence(audit, "started_at")
    check_timestamp(audit["started_at"])

    check_attribute_presence(audit, "ended_at")
    check_timestamp(audit["ended_at"])

def get_details_node(context):
    """Get content of details node, given it exists."""
    data = context.s3_data

    return check_and_get_attribute(data, 'details')

def check_cve_value(cve, with_score=False):
    """Check CVE values in CVE records."""
    if with_score:
        # please note that in graph DB, the CVE entries have the following format:
        # CVE-2012-1150:5.0
        # don't ask me why, but the score is stored in one field together with ID itself
        # the : character is used as a separator
        pattern = "CVE-(\d{4})-\d{4,}:(\d+\.\d+)"
    else:
        pattern = "CVE-(\d{4})-\d{4,}"

    match = re.fullmatch(pattern, cve)
    assert match is not None, "Improper CVE number %s" % cve

    year = int(match.group(1))
    current_year = datetime.datetime.now().year

    # well the lower limit is a bit arbitrary
    # (according to SRT guys it should be 1999)
    assert year >= 1999 and year <= current_year

    if with_score:
        score = float(match.group(2))
        assert score >= 0.0 and score <= 10.0
<commit_msg>Function to parse major, minor, patch numbers from version string<commit_after>"""Utility functions to check attributes returned in API responses and read from the AWS S3."""
import datetime
import re

def check_attribute_presence(node, attribute_name):
    """Check the attribute presence in the given dictionary or list.

    To be used to check the deserialized JSON data etc.
    """
    found_attributes = node if type(node) is list else node.keys()
    assert attribute_name in node, \
        "'%s' attribute is expected in the node, " \
        "found: %s attributes " % (attribute_name, ", ".join(found_attributes))

def check_attributes_presence(node, attribute_names):
    """Check the presence of all attributes in the dictionary or in the list.

    To be used to check the deserialized JSON data etc.
    """
    for attribute_name in attribute_names:
        found_attributes = node if type(node) is list else node.keys()
        assert attribute_name in node, \
            "'%s' attribute is expected in the node, " \
            "found: %s attributes " % (attribute_name, ", ".join(found_attributes))

def check_and_get_attribute(node, attribute_name):
    """Check the attribute presence and if the attribute is found, return its value."""
    check_attribute_presence(node, attribute_name)
    return node[attribute_name]

def check_uuid(uuid):
    """Check if the string contains a proper UUID.

    Supported format: 71769af6-0a39-4242-94be-1f84f04c8a56
    """
    regex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}\Z',
                       re.I)
    match = regex.match(uuid)
    return bool(match)

def check_timestamp(timestamp):
    """Check if the string contains proper timestamp value.

    The following four formats are supported:
    2017-07-19 13:05:25.041688
    2017-07-17T09:05:29.101780
    2017-07-19 13:05:25
    2017-07-17T09:05:29
    """
    assert timestamp is not None
    assert isinstance(timestamp, str)

    # some attributes contains timestamp without the millisecond part
    # so we need to take care of it
    if len(timestamp) == len("YYYY-mm-dd HH:MM:SS") and '.' not in timestamp:
        timestamp += '.0'

    assert len(timestamp) >= len("YYYY-mm-dd HH:MM:SS.")

    # we have to support the following formats:
    #    2017-07-19 13:05:25.041688
    #    2017-07-17T09:05:29.101780
    # -> it is needed to distinguish the 'T' separator
    #
    # (please see https://www.tutorialspoint.com/python/time_strptime.htm for
    #  an explanation how timeformat should look like)

    timeformat = "%Y-%m-%d %H:%M:%S.%f"
    if timestamp[10] == "T":
        timeformat = "%Y-%m-%dT%H:%M:%S.%f"

    # just try to parse the string to check whether
    # the ValueError exception is raised or not
    datetime.datetime.strptime(timestamp, timeformat)

def check_job_token_attributes(token):
    """Check that the given JOB token contains all required attributes."""
    attribs = ["limit", "remaining", "reset"]
    for attr in attribs:
        assert attr in token
        assert int(token[attr]) >= 0

def check_status_attribute(data):
    """Check the value of the status attribute, that should contain just two allowed values."""
    status = check_and_get_attribute(data, "status")
    assert status in ["success", "error"]

def check_summary_attribute(data):
    """Check the summary attribute that can be found all generated metadata."""
    summary = check_and_get_attribute(data, "summary")
    assert type(summary) is list or type(summary) is dict

def release_string(ecosystem, package, version=None):
    """Construct a string with ecosystem:package or ecosystem:package:version tuple."""
    return "{e}:{p}:{v}".format(e=ecosystem, p=package, v=version)

def check_release_attribute(data, ecosystem, package, version=None):
    """Check the content of _release attribute.

    Check that the attribute _release contains proper release string for given ecosystem
    and package.
    """
    check_attribute_presence(data, "_release")
    assert data["_release"] == release_string(ecosystem, package, version)

def check_schema_attribute(data, expected_schema_name, expected_schema_version):
    """Check the content of the schema attribute.

    This attribute should contains dictionary with name and version that are checked as well.
    """
    # read the toplevel attribute 'schema'
    schema = check_and_get_attribute(data, "schema")

    # read attributes from the 'schema' node
    name = check_and_get_attribute(schema, "name")
    version = check_and_get_attribute(schema, "version")

    # check the schema name
    assert name == expected_schema_name, "Schema name '{n1}' is different from " \
        "expected name '{n2}'".format(n1=name, n2=expected_schema_name)

    # check the schema version (ATM we are able to check just one fixed version)
    assert version == expected_schema_version, "Schema version {v1} is different from expected " \
        "version {v2}".format(v1=version, v2=expected_schema_version)

def check_audit_metadata(data):
    """Check the metadata stored in the _audit attribute.

    Check if all common attributes can be found in the audit node
    in the component or package metadata.
    """
    check_attribute_presence(data, "_audit")
    audit = data["_audit"]

    check_attribute_presence(audit, "version")
    assert audit["version"] == "v1"

    check_attribute_presence(audit, "started_at")
    check_timestamp(audit["started_at"])

    check_attribute_presence(audit, "ended_at")
    check_timestamp(audit["ended_at"])

def get_details_node(context):
    """Get content of details node, given it exists."""
    data = context.s3_data

    return check_and_get_attribute(data, 'details')

def check_cve_value(cve, with_score=False):
    """Check CVE values in CVE records."""
    if with_score:
        # please note that in graph DB, the CVE entries have the following format:
        # CVE-2012-1150:5.0
        # don't ask me why, but the score is stored in one field together with ID itself
        # the : character is used as a separator
        pattern = "CVE-(\d{4})-\d{4,}:(\d+\.\d+)"
    else:
        pattern = "CVE-(\d{4})-\d{4,}"

    match = re.fullmatch(pattern, cve)
    assert match is not None, "Improper CVE number %s" % cve

    year = int(match.group(1))
    current_year = datetime.datetime.now().year

    # well the lower limit is a bit arbitrary
    # (according to SRT guys it should be 1999)
    assert year >= 1999 and year <= current_year

    if with_score:
        score = float(match.group(2))
        assert score >= 0.0 and score <= 10.0

def parse_version(value):
    """Parse the version string and split it into major, minor, patch numbers."""
    match = re.match("(\d)\.(\d)\.(\d).*", value)
    return int(match.group(1)), int(match.group(2)), int(match.group(3))
<|endoftext|>