<commit_before># fMBT, free Model Based Testing tool
# Copyright (c) 2014, Intel Corporation.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms and conditions of the GNU Lesser General Public License,
# version 2.1, as published by the Free Software Foundation.
#
# This program is distributed in the hope it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
# License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA
# 02110-1301 USA.

# Converts raw images into PNG, based on libpng.

import ctypes
try:
    import fmbtgti
except ImportError:
    fmbtgti = None

PNG_MAGIC = "\x89PNG\x0d\x0a\x1a\x0a"
PNG_HEADER_VERSION_STRING = None
PNG_LIBPNG_VERSION_STRING = None
PNG_COLOR_MASK_COLOR = 2
PNG_COLOR_MASK_ALPHA = 4
PNG_COLOR_TYPE_RGB = PNG_COLOR_MASK_COLOR
PNG_COLOR_TYPE_RGB_ALPHA = PNG_COLOR_MASK_COLOR | PNG_COLOR_MASK_ALPHA
PNG_INTERLACE_NONE = 0
PNG_COMPRESSION_TYPE_DEFAULT = 0
PNG_FILTER_TYPE_DEFAULT = 0
PNG_TRANSFORM_IDENTITY = 0

NULL = ctypes.c_void_p(0)

libpng = None
for libpng_filename in ["libpng.so", "libpng12.so.0", "libpng15.so.15",
                        "libpng16.so.16"]:
    try:
        libpng = ctypes.CDLL(libpng_filename)
        break
    except OSError:
        continue
if libpng == None:
    raise ImportError("loading libpng.so failed")

libpng.png_create_write_struct.restype = ctypes.c_void_p
libpng.png_create_write_struct.argtypes = [
    ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]

libpng.png_get_header_ver.restype = ctypes.c_char_p
libpng.png_get_header_ver.argtypes = [ctypes.c_void_p]

libpng.png_create_info_struct.restype = ctypes.c_void_p
libpng.png_create_info_struct.argtypes = [ctypes.c_void_p]

libpng.png_set_write_fn.restype = None
libpng.png_set_write_fn.argtypes = [
    ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]

libpng.png_set_rows.restype = None
libpng.png_set_rows.argtypes = [
    ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]

libpng.png_set_IHDR.restype = None
libpng.png_set_IHDR.argtypes = [
    ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32,
    ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int]

libpng.png_write_png.restype = None
libpng.png_write_png.argtypes = [
    ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p]

PNG_LIBPNG_VERSION_STRING = libpng.png_get_header_ver(None)

def raw2png(data, width, height, depth=8, fmt="RGB"):
    """convert raw image into PNG

    Parameters:
      data (string or a ctypes pointer type):
              image data

      width (integer):
              width of the image in pixels

      height (integer):
              height of the image in pixels

      depth (integer, optional):
              color depth, bits per color channel.
              The default is 8 (that is, RGB has 24 bpp, RGBA 32 bpp).

      fmt (string, optional):
              image data format. The default is "RGB".
              Supported formats: "RGB", "RGBA", "RGB_", "BGR", "BGR_".

    Returns string that contains PNG image data.

    Example: create a png image with one half-opaque green pixel
      png_data = raw2png("\x00\xff\x00\x80", 1, 1, fmt="RGBA")
      file("green.png", "wb").write(png_data)
    """
    png_data = []

    png_struct = ctypes.c_void_p(
        libpng.png_create_write_struct(PNG_LIBPNG_VERSION_STRING,
                                       NULL, NULL, NULL))

    if not png_struct:
        raise PngError("png_create_write_struct failed")

    info_struct = ctypes.c_void_p(
        libpng.png_create_info_struct(png_struct))
    if not info_struct:
        libpng.png_destroy_write_struct(png_struct, ctypes.c_void_p(0))
        raise PngError("png_create_info_struct failed")

    def cb_png_write(png_struct, data, datalen):
        png_data.append(ctypes.string_at(data, datalen))
    c_cb_png_write = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p,
                                      ctypes.c_size_t)(cb_png_write)

    libpng.png_set_write_fn(png_struct, NULL, c_cb_png_write, NULL)

    if isinstance(data, str):
        buf = ctypes.c_buffer(data)
    else:
        buf = data.contents

    fmt = fmt.upper()
    if fmt == "RGB":
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = (depth / 8) * 3
    elif fmt == "RGB565":
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = 3
        depth = 8
        orig_buf = buf
        buf = ctypes.create_string_buffer(width * height * bytes_per_pixel)
        fmbtgti.eye4graphics.rgb5652rgb(orig_buf, width, height, buf)
    elif fmt == "RGBA":
        color_type = PNG_COLOR_TYPE_RGB_ALPHA
        bytes_per_pixel = (depth / 8) * 4
    elif fmt == "RGB_" and fmbtgti:
        fmbtgti.eye4graphics.rgbx2rgb(buf, width, height)
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = (depth / 8) * 3
    elif fmt == "BGR" and fmbtgti:
        fmbtgti.eye4graphics.bgr2rgb(buf, width, height)
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = (depth / 8) * 3
    elif fmt == "BGR_" and fmbtgti:
        fmbtgti.eye4graphics.bgrx2rgb(buf, width, height)
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = (depth / 8) * 3
    else:
        raise ValueError('Unsupported data format "%s", use "RGB" or "RGBA"')

    buf_addr = ctypes.addressof(buf)

    libpng.png_set_IHDR(
        png_struct, info_struct, width, height, depth,
        color_type, PNG_INTERLACE_NONE,
        PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT)

    rows = (ctypes.c_void_p * height)()
    bytes_per_row = width * bytes_per_pixel
    for row in xrange(height):
        rows[row] = buf_addr
        buf_addr += bytes_per_row

    libpng.png_set_rows(png_struct, info_struct, rows)
    libpng.png_write_png(png_struct, info_struct, PNG_TRANSFORM_IDENTITY, NULL)

    png_structp = ctypes.POINTER(ctypes.c_void_p)(png_struct)
    png_infop = ctypes.POINTER(ctypes.c_void_p)(info_struct)

    libpng.png_destroy_write_struct(png_structp, png_infop)

    return "".join(png_data)

class PngError(Exception):
    pass

if __name__ == "__main__":
    png_data = raw2png(
        "\xff\x00\x00\x80\x00\x00\x40\x00\x00\x20\x00\x00"
        "\x00\xff\x00\x00\x80\x00\x00\x40\x00\x00\x20\x00"
        "\x00\x00\xff\x00\x00\x80\x00\x00\x40\x00\x00\x20",
        4, 3, 8, "rgb")
    file("fmbtpng-test.png", "wb").write(png_data)
<commit_msg>fmbtpng: added windows support<commit_after># fMBT, free Model Based Testing tool
# Copyright (c) 2014, Intel Corporation.
#
# This program is free software; you can redistribute it and/or modify it
# under the terms and conditions of the GNU Lesser General Public License,
# version 2.1, as published by the Free Software Foundation.
#
# This program is distributed in the hope it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
# License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA
# 02110-1301 USA.

# Converts raw images into PNG, based on libpng.

import ctypes
import os

try:
    import fmbtgti
except ImportError:
    fmbtgti = None

PNG_MAGIC = "\x89PNG\x0d\x0a\x1a\x0a"
PNG_HEADER_VERSION_STRING = None
PNG_LIBPNG_VERSION_STRING = None
PNG_COLOR_MASK_COLOR = 2
PNG_COLOR_MASK_ALPHA = 4
PNG_COLOR_TYPE_RGB = PNG_COLOR_MASK_COLOR
PNG_COLOR_TYPE_RGB_ALPHA = PNG_COLOR_MASK_COLOR | PNG_COLOR_MASK_ALPHA
PNG_INTERLACE_NONE = 0
PNG_COMPRESSION_TYPE_DEFAULT = 0
PNG_FILTER_TYPE_DEFAULT = 0
PNG_TRANSFORM_IDENTITY = 0

NULL = ctypes.c_void_p(0)

if os.name == "nt":
    libpng_filenames = ["libpng16-16.dll"]
else:
    libpng_filenames = ["libpng.so", "libpng12.so.0", "libpng15.so.15",
                        "libpng16.so.16"]

libpng = None
for libpng_filename in libpng_filenames:
    try:
        libpng = ctypes.CDLL(libpng_filename)
        break
    except OSError:
        continue
if libpng == None:
    raise ImportError("loading libpng.so failed")

libpng.png_create_write_struct.restype = ctypes.c_void_p
libpng.png_create_write_struct.argtypes = [
    ctypes.c_char_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]

libpng.png_get_header_ver.restype = ctypes.c_char_p
libpng.png_get_header_ver.argtypes = [ctypes.c_void_p]

libpng.png_create_info_struct.restype = ctypes.c_void_p
libpng.png_create_info_struct.argtypes = [ctypes.c_void_p]

libpng.png_set_write_fn.restype = None
libpng.png_set_write_fn.argtypes = [
    ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]

libpng.png_set_rows.restype = None
libpng.png_set_rows.argtypes = [
    ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p]

libpng.png_set_IHDR.restype = None
libpng.png_set_IHDR.argtypes = [
    ctypes.c_void_p, ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32,
    ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int, ctypes.c_int]

libpng.png_write_png.restype = None
libpng.png_write_png.argtypes = [
    ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p]

PNG_LIBPNG_VERSION_STRING = libpng.png_get_header_ver(None)

def raw2png(data, width, height, depth=8, fmt="RGB"):
    """convert raw image into PNG

    Parameters:
      data (string or a ctypes pointer type):
              image data

      width (integer):
              width of the image in pixels

      height (integer):
              height of the image in pixels

      depth (integer, optional):
              color depth, bits per color channel.
              The default is 8 (that is, RGB has 24 bpp, RGBA 32 bpp).

      fmt (string, optional):
              image data format. The default is "RGB".
              Supported formats: "RGB", "RGBA", "RGB_", "BGR", "BGR_".

    Returns string that contains PNG image data.

    Example: create a png image with one half-opaque green pixel
      png_data = raw2png("\x00\xff\x00\x80", 1, 1, fmt="RGBA")
      file("green.png", "wb").write(png_data)
    """
    png_data = []

    png_struct = ctypes.c_void_p(
        libpng.png_create_write_struct(PNG_LIBPNG_VERSION_STRING,
                                       NULL, NULL, NULL))

    if not png_struct:
        raise PngError("png_create_write_struct failed")

    info_struct = ctypes.c_void_p(
        libpng.png_create_info_struct(png_struct))
    if not info_struct:
        libpng.png_destroy_write_struct(png_struct, ctypes.c_void_p(0))
        raise PngError("png_create_info_struct failed")

    def cb_png_write(png_struct, data, datalen):
        png_data.append(ctypes.string_at(data, datalen))
    c_cb_png_write = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_void_p,
                                      ctypes.c_size_t)(cb_png_write)

    libpng.png_set_write_fn(png_struct, NULL, c_cb_png_write, NULL)

    if isinstance(data, str):
        buf = ctypes.c_buffer(data)
    else:
        buf = data.contents

    fmt = fmt.upper()
    if fmt == "RGB":
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = (depth / 8) * 3
    elif fmt == "RGB565":
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = 3
        depth = 8
        orig_buf = buf
        buf = ctypes.create_string_buffer(width * height * bytes_per_pixel)
        fmbtgti.eye4graphics.rgb5652rgb(orig_buf, width, height, buf)
    elif fmt == "RGBA":
        color_type = PNG_COLOR_TYPE_RGB_ALPHA
        bytes_per_pixel = (depth / 8) * 4
    elif fmt == "RGB_" and fmbtgti:
        fmbtgti.eye4graphics.rgbx2rgb(buf, width, height)
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = (depth / 8) * 3
    elif fmt == "BGR" and fmbtgti:
        fmbtgti.eye4graphics.bgr2rgb(buf, width, height)
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = (depth / 8) * 3
    elif fmt == "BGR_" and fmbtgti:
        fmbtgti.eye4graphics.bgrx2rgb(buf, width, height)
        color_type = PNG_COLOR_TYPE_RGB
        bytes_per_pixel = (depth / 8) * 3
    else:
        raise ValueError('Unsupported data format "%s", use "RGB" or "RGBA"')

    buf_addr = ctypes.addressof(buf)

    libpng.png_set_IHDR(
        png_struct, info_struct, width, height, depth,
        color_type, PNG_INTERLACE_NONE,
        PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT)

    rows = (ctypes.c_void_p * height)()
    bytes_per_row = width * bytes_per_pixel
    for row in xrange(height):
        rows[row] = buf_addr
        buf_addr += bytes_per_row

    libpng.png_set_rows(png_struct, info_struct, rows)
    libpng.png_write_png(png_struct, info_struct, PNG_TRANSFORM_IDENTITY, NULL)

    png_structp = ctypes.POINTER(ctypes.c_void_p)(png_struct)
    png_infop = ctypes.POINTER(ctypes.c_void_p)(info_struct)

    libpng.png_destroy_write_struct(png_structp, png_infop)

    return "".join(png_data)

class PngError(Exception):
    pass

if __name__ == "__main__":
    png_data = raw2png(
        "\xff\x00\x00\x80\x00\x00\x40\x00\x00\x20\x00\x00"
        "\x00\xff\x00\x00\x80\x00\x00\x40\x00\x00\x20\x00"
        "\x00\x00\xff\x00\x00\x80\x00\x00\x40\x00\x00\x20",
        4, 3, 8, "rgb")
    file("fmbtpng-test.png", "wb").write(png_data)
<|endoftext|>


<commit_before>#!/usr/bin/python

import urwid
import urwid.curses_display

"""
NOTES
-----
These are the possible colors we can use:
black, dark red, dark green, brown, dark blue, dark magenta, dark cyan, light gray

"""

def exit_loop(key):
  if key in ('q', 'Q'):
    raise urwid.ExitMainLoop()

def main():
  text_header = (u" Welcome to our CS419 project! q or Q exits the program.")
  text_leftcol1 = (u"Top of left column")
  text_leftcol2 = (u"This is the body of the left column")
  text_maintop = (u"This is the top section of the main body")
  text_mainbody = (u" This is the main body section")
  text_instructions = (u"This program allows you to connect to a PostgreSQL or MySQL database and then perform operations on that databse. The program is written in python and is an ncurses based command line tool.")

  blank = urwid.Divider()
  listbox_content = [
    blank,
    urwid.Padding(urwid.Text(text_instructions), left=2, right=2),
    blank,
    urwid.Columns([
      ('fixed', 15,
        urwid.AttrWrap( urwid.Padding( urwid.Pile([
              urwid.Text(text_leftcol1),
              urwid.Divider("~"),
              urwid.Text(text_leftcol2)
            ])
          , left=1, right=1)
        , 'leftside')
      ),
      urwid.AttrWrap( urwid.Padding( urwid.Pile([
            urwid.AttrWrap( urwid.Text(text_maintop), 'topmenu'),
            urwid.AttrWrap( urwid.Divider("-"), 'topmenu'),
            blank,
            urwid.Text(text_mainbody),
            urwid.Text(u" It can keep going down"),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ...")
          ])
        , left=0, right=1)
      , 'body')
    ])
  ]

  frame_header = urwid.AttrWrap(urwid.Text(text_header), 'header')
  listbox = urwid.ListBox(urwid.SimpleListWalker(listbox_content))
  frame = urwid.Frame(urwid.AttrWrap(listbox, 'bg'), header=frame_header)

  palette = [
      ('header', 'light gray', 'dark red'),
      ('topmenu', 'light gray', 'dark blue'),
      ('leftside', 'black', 'dark cyan'),
      ('body', '', 'black'),
      ('bg', 'black', 'light gray')
  ]

  loop = urwid.MainLoop(frame, palette, unhandled_input=exit_loop, screen=urwid.curses_display.Screen())

  loop.run()

if __name__ == '__main__':
  main()
<commit_msg>fixed formatting for left side menu with buttons now<commit_after>#!/usr/bin/python

import urwid
import urwid.curses_display

"""
NOTES
-----
These are the possible colors we can use:
black, dark red, dark green, brown, dark blue, dark magenta, dark cyan, light gray

"""

def exit_loop(key):
  if key in ('q', 'Q'):
    raise urwid.ExitMainLoop()

def main():
  text_header = (u" Welcome to our CS419 project! q or Q exits the program.")
  text_leftcol1_1 = (u"Database:")
  text_leftcol1_2 = (u"Test DB")
  text_leftcol2_1 = (u"Tables:")
  text_leftcol2_2 = (u"Table 1")
  text_leftcol2_3 = (u"Table 2")
  text_leftcol2_4 = (u"Table 3")
  text_leftcol2_5 = (u"Table 4")
  text_leftcol2_6 = (u"Table 5")
  text_leftcol2_7 = (u"Table 6")
  text_maintop = (u"This is the top section of the main body")
  text_mainbody = (u" This is the main body section")
  text_instructions = (u"This program allows you to connect to a PostgreSQL or MySQL database and then perform operations on that databse. The program is written in python and is an ncurses based command line tool.")

  #this creates blank as a simple divider widget to add a line of space
  blank = urwid.Divider()

  #Signal handler for the left side buttons
  def leftcol_btn_press(button):
      frame.footer = urwid.AttrWrap(urwid.Text(
          [u" Pressed: ", button.get_label()]), 'header')

  #creating left column widget
  left_column = urwid.AttrWrap( urwid.Padding (urwid.Pile([
          blank,
          urwid.Text(text_leftcol1_1, align='center'),
          urwid.AttrWrap( urwid.Button(text_leftcol1_2, leftcol_btn_press), 'btn', 'btnf'),
          urwid.Divider("="),
          blank,
          urwid.Text(text_leftcol2_1, align='center'),
          urwid.AttrWrap( urwid.Button(text_leftcol2_2, leftcol_btn_press), 'btn', 'btnf'),
          urwid.AttrWrap( urwid.Button(text_leftcol2_3, leftcol_btn_press), 'btn', 'btnf'),
          urwid.AttrWrap( urwid.Button(text_leftcol2_4, leftcol_btn_press), 'btn', 'btnf'),
          urwid.AttrWrap( urwid.Button(text_leftcol2_5, leftcol_btn_press), 'btn', 'btnf'),
          urwid.AttrWrap( urwid.Button(text_leftcol2_6, leftcol_btn_press), 'btn', 'btnf'),
          urwid.AttrWrap( urwid.Button(text_leftcol2_7, leftcol_btn_press), 'btn', 'btnf'),
          blank
        ])
      , left=1, right=2)
    , 'leftside')

  listbox_content = [
    blank,
    urwid.Padding(urwid.Text(text_instructions), left=2, right=2),
    blank,
    urwid.Columns([
      ('fixed', 17, left_column),
      urwid.AttrWrap( urwid.Padding( urwid.Pile([
            urwid.AttrWrap( urwid.Text(text_maintop), 'topmenu'),
            urwid.AttrWrap( urwid.Divider("-"), 'topmenu'),
            blank,
            urwid.Text(text_mainbody),
            urwid.Text(u" It can keep going down"),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ..."),
            urwid.Text(u" ... ... ... ... ... ... ... ... ... ... ...")
          ])
        , left=0, right=1)
      , 'body')
    ])
  ]

  frame_header = urwid.AttrWrap(urwid.Text(text_header), 'header')
  listbox = urwid.ListBox(urwid.SimpleListWalker(listbox_content))
  frame = urwid.Frame(urwid.AttrWrap(listbox, 'bg'), header=frame_header)

  palette = [
      ('header', 'light gray', 'dark red'),
      ('topmenu', 'light gray', 'dark blue'),
      ('leftside', 'light gray', 'dark cyan'),
      ('body', 'black', 'light gray'),
      ('bg', 'black', 'light gray'),
      ('btn','light gray','dark cyan'),
      ('btnf','light gray','dark blue')
  ]

  loop = urwid.MainLoop(frame, palette, unhandled_input=exit_loop, screen=urwid.curses_display.Screen())

  loop.run()

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Models for the FOIA application
"""

import datetime

from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.core.validators import validate_email
from django.db import models
from django.shortcuts import get_object_or_404

import email
import logging

from muckrock.foia.models.request import FOIARequest, STATUS

logger = logging.getLogger(__name__)

DELIVERED = (
    ('fax', 'Fax'),
    ('email', 'Email'),
    ('mail', 'Mail'),
)

def requests_from_pks(foia_pks):
    """A helper function to get all the requests given a list of their PKs."""
    foias = []
    # if foia_pks isn't a list (say, a single pk), then it should be made into one
    if not isinstance(foia_pks, list):
        foia_pks = [foia_pks]
    for foia_pk in foia_pks:
        try:
            foia = FOIARequest.objects.get(pk=foia_pk)
            foias.append(foia)
        except (FOIARequest.DoesNotExist, ValueError):
            logging.error('FOIA %s does not exist', foia_pk)
            continue
    return foias

class FOIACommunication(models.Model):
    """A single communication of a FOIA request"""

    foia = models.ForeignKey(FOIARequest, related_name='communications', blank=True, null=True)
    from_who = models.CharField(max_length=255)
    to_who = models.CharField(max_length=255, blank=True)
    priv_from_who = models.CharField(max_length=255, blank=True)
    priv_to_who = models.CharField(max_length=255, blank=True)
    subject = models.CharField(max_length=255, blank=True)
    date = models.DateTimeField(db_index=True)
    response = models.BooleanField(default=False,
            help_text='Is this a response (or a request)?')
    autogenerated = models.BooleanField(default=False)
    full_html = models.BooleanField(default=False)
    communication = models.TextField(blank=True)
    delivered = models.CharField(max_length=10, choices=DELIVERED, blank=True, null=True)
    # what status this communication should set the request to - used for machine learning
    status = models.CharField(max_length=10, choices=STATUS, blank=True, null=True)
    opened = models.BooleanField(default=False,
            help_text='If emailed, did we receive an open notification? '
                      'If faxed, did we recieve a confirmation?')

    # only used for orphans
    likely_foia = models.ForeignKey(
        FOIARequest,
        related_name='likely_communications',
        blank=True,
        null=True
    )

    reindex_related = ('foia',)

    def __unicode__(self):
        return '%s: %s...' % (self.date.strftime('%m/%d/%y'), self.communication[:80])

    def get_absolute_url(self):
        """The url for this object"""
        # pylint: disable=no-member
        return self.foia.get_absolute_url() + ('#comm-%d' % self.pk)

    def save(self, *args, **kwargs):
        """Remove controls characters from text before saving"""
        remove_control = dict.fromkeys(range(0, 9) + range(11, 13) + range(14, 32))
        self.communication = unicode(self.communication).translate(remove_control)
        # limit communication length to 150k
        self.communication = self.communication[:150000]
        # special handling for certain agencies
        self._presave_special_handling()
        super(FOIACommunication, self).save(*args, **kwargs)

    def anchor(self):
        """Anchor name"""
        return 'comm-%d' % self.pk

    def move(self, foia_pks):
        """
        Move this communication. If more than one foia_pk is given, move the
        communication to the first request, then clone it across the rest of
        the requests. Returns the moved and cloned communications.
        """
        if not foia_pks:
            raise ValueError('Expected a request to move the communication to.')
        if not isinstance(foia_pks, list):
            foia_pks = [foia_pks]
        move_to_request = get_object_or_404(FOIARequest, pk=foia_pks[0])
        self.foia = move_to_request
        for each_file in self.files.all():
            each_file.foia = move_to_request
            each_file.save()
        self.save()
        logging.info('Communication #%d moved to request #%d', self.id, self.foia.id)
        # if cloning happens, self gets overwritten. so we save it to a variable here
        this_comm = FOIACommunication.objects.get(pk=self.pk)
        moved = [this_comm]
        cloned = []
        if foia_pks[1:]:
            cloned = self.clone(foia_pks[1:])
        return moved + cloned

    def clone(self, foia_pks):
        """
        Copies the communication to each request in the list,
        then returns all the new communications.
        ---
        When setting self.pk to None and then calling self.save(),
        Django will clone the communication along with all of its data
        and give it a new primary key. On the next iteration of the loop,
        the clone will be cloned along with its data, and so on. Same thing
        goes for each file attached to the communication.
        """
        # avoid circular imports
        from muckrock.foia.tasks import upload_document_cloud
        request_list = requests_from_pks(foia_pks)
        if not request_list:
            raise ValueError('No valid request(s) provided for cloning.')
        cloned_comms = []
        original_pk = self.pk
        files = self.files.all()
        for request in request_list:
            this_clone = FOIACommunication.objects.get(pk=original_pk)
            this_clone.pk = None
            this_clone.foia = request
            this_clone.save()
            for file_ in files:
                original_file_id = file_.id
                file_.pk = None
                file_.foia = request
                file_.comm = this_clone
                # make a copy of the file on the storage backend
                try:
                    new_ffile = ContentFile(file_.ffile.read())
                except ValueError:
                    error_msg = ('FOIAFile #%s has no data in its ffile field. '
                                'It has not been cloned.')
                    logging.error(error_msg, original_file_id)
                    continue
                new_ffile.name = file_.ffile.name
                file_.ffile = new_ffile
                file_.save()
                upload_document_cloud.apply_async(args=[file_.pk, False], countdown=3)
            # for each clone, self gets overwritten. each clone needs to be stored explicitly.
            cloned_comms.append(this_clone)
            logging.info('Communication #%d cloned to request #%d', original_pk, this_clone.foia.id)
        return cloned_comms

    def resend(self, email_address=None):
        """Resend the communication"""
        foia = self.foia
        if not foia:
            logging.error('Tried resending an orphaned communication.')
            raise ValueError('This communication has no FOIA to submit.')
        snail = False
        self.date = datetime.datetime.now()
        self.save()
        if email_address:
            # responsibility for handling validation errors
            # is on the caller of the resend method
            validate_email(email_address)
            foia.email = email_address
            foia.save()
        else:
            snail = True
        foia.submit(snail=snail)
        logging.info('Communication #%d resent.', self.id)

    def set_raw_email(self, msg):
        """Set the raw email for this communication"""
        raw_email = RawEmail.objects.get_or_create(communication=self)[0]
        raw_email.raw_email = msg
        raw_email.save()

    def get_sender_email(self):
        """Get the email this communication was sent from."""
        _, sender_email = email.utils.parseaddr(self.priv_from_who)
        try:
            validate_email(sender_email)
        except ValidationError:
            # if the sender email is invalid, don't return anything
            sender_email = None
        return sender_email

    def make_sender_primary_contact(self):
        """Makes the communication's sender the primary email of its FOIA."""
        sender_email = self.get_sender_email()
        if not sender_email:
            raise ValueError('Communication was not sent from a valid email.')
        if not self.foia:
            raise ValueError('Communication is an orphan and has no associated request.')
        self.foia.email = sender_email
        self.foia.save()
        return

    def _presave_special_handling(self):
        """Special handling before saving
        For example, strip out BoP excessive quoting"""

        def test_agency_name(name):
            """Match on agency name"""
            return (self.foia and self.foia.agency and
                    self.foia.agency.name == name)

        def until_string(string):
            """Cut communication off after string"""
            def modify():
                """Run the modification on self.communication"""
                if string in self.communication:
                    idx = self.communication.index(string)
                    self.communication = self.communication[:idx]
            return modify

        special_cases = [
            # BoP: strip everything after '>>>'
            (test_agency_name('Bureau of Prisons'),
             until_string('>>>')),
            # Phoneix Police: strip everything after '_'*32
            (test_agency_name('Phoenix Police Department'),
             until_string('_' * 32)),
        ]

        for test, modify in special_cases:
            if test:
                modify()

    class Meta:
        # pylint: disable=too-few-public-methods
        ordering = ['date']
        verbose_name = 'FOIA Communication'
        app_label = 'foia'

class RawEmail(models.Model):
    """The raw email text for a communication - stored seperately for performance"""

    communication = models.OneToOneField(FOIACommunication)
    raw_email = models.TextField(blank=True)

    def __unicode__(self):
        return 'Raw Email: %d' % self.pk

    class Meta:
        app_label = 'foia'

class FOIANote(models.Model):
    """A private note on a FOIA request"""

    foia = models.ForeignKey(FOIARequest, related_name='notes')
    author = models.ForeignKey('auth.User', related_name='notes', null=True)
    datetime = models.DateTimeField()
    note = models.TextField()

    def __unicode__(self):
        # pylint: disable=no-member
        return 'Note by %s on %s' % (self.author.get_full_name(), self.foia.title)

    class Meta:
        # pylint: disable=too-few-public-methods
        ordering = ['foia', 'datetime']
        verbose_name = 'FOIA Note'
        app_label = 'foia'
<commit_msg>fixes bug where unicode would crash when note had no author<commit_after># -*- coding: utf-8 -*-
"""
Models for the FOIA application
"""

import datetime

from django.core.exceptions import ValidationError
from django.core.files.base import ContentFile
from django.core.validators import validate_email
from django.db import models
from django.shortcuts import get_object_or_404

import email
import logging

from muckrock.foia.models.request import FOIARequest, STATUS

logger = logging.getLogger(__name__)

DELIVERED = (
    ('fax', 'Fax'),
    ('email', 'Email'),
    ('mail', 'Mail'),
)

def requests_from_pks(foia_pks):
    """A helper function to get all the requests given a list of their PKs."""
    foias = []
    # if foia_pks isn't a list (say, a single pk), then it should be made into one
    if not isinstance(foia_pks, list):
        foia_pks = [foia_pks]
    for foia_pk in foia_pks:
        try:
            foia = FOIARequest.objects.get(pk=foia_pk)
            foias.append(foia)
        except (FOIARequest.DoesNotExist, ValueError):
            logging.error('FOIA %s does not exist', foia_pk)
            continue
    return foias

class FOIACommunication(models.Model):
    """A single communication of a FOIA request"""

    foia = models.ForeignKey(FOIARequest, related_name='communications', blank=True, null=True)
    from_who = models.CharField(max_length=255)
    to_who = models.CharField(max_length=255, blank=True)
    priv_from_who = models.CharField(max_length=255, blank=True)
    priv_to_who = models.CharField(max_length=255, blank=True)
    subject = models.CharField(max_length=255, blank=True)
    date = models.DateTimeField(db_index=True)
    response = models.BooleanField(default=False,
            help_text='Is this a response (or a request)?')
    autogenerated = models.BooleanField(default=False)
    full_html = models.BooleanField(default=False)
    communication = models.TextField(blank=True)
    delivered = models.CharField(max_length=10, choices=DELIVERED, blank=True, null=True)
    # what status this communication should set the request to - used for machine learning
    status = models.CharField(max_length=10, choices=STATUS, blank=True, null=True)
    opened = models.BooleanField(default=False,
            help_text='If emailed, did we receive an open notification? '
                      'If faxed, did we recieve a confirmation?')

    # only used for orphans
    likely_foia = models.ForeignKey(
        FOIARequest,
        related_name='likely_communications',
        blank=True,
        null=True
    )

    reindex_related = ('foia',)

    def __unicode__(self):
        return '%s: %s...' % (self.date.strftime('%m/%d/%y'), self.communication[:80])

    def get_absolute_url(self):
        """The url for this object"""
        # pylint: disable=no-member
        return self.foia.get_absolute_url() + ('#comm-%d' % self.pk)

    def save(self, *args, **kwargs):
        """Remove controls characters from text before saving"""
        remove_control = dict.fromkeys(range(0, 9) + range(11, 13) + range(14, 32))
        self.communication = unicode(self.communication).translate(remove_control)
        # limit communication length to 150k
        self.communication = self.communication[:150000]
        # special handling for certain agencies
        self._presave_special_handling()
        super(FOIACommunication, self).save(*args, **kwargs)

    def anchor(self):
        """Anchor name"""
        return 'comm-%d' % self.pk

    def move(self, foia_pks):
        """
        Move this communication. If more than one foia_pk is given, move the
        communication to the first request, then clone it across the rest of
        the requests. Returns the moved and cloned communications.
        """
        if not foia_pks:
            raise ValueError('Expected a request to move the communication to.')
        if not isinstance(foia_pks, list):
            foia_pks = [foia_pks]
        move_to_request = get_object_or_404(FOIARequest, pk=foia_pks[0])
        self.foia = move_to_request
        for each_file in self.files.all():
            each_file.foia = move_to_request
            each_file.save()
        self.save()
        logging.info('Communication #%d moved to request #%d', self.id, self.foia.id)
        # if cloning happens, self gets overwritten. so we save it to a variable here
        this_comm = FOIACommunication.objects.get(pk=self.pk)
        moved = [this_comm]
        cloned = []
        if foia_pks[1:]:
            cloned = self.clone(foia_pks[1:])
        return moved + cloned

    def clone(self, foia_pks):
        """
        Copies the communication to each request in the list,
        then returns all the new communications.
        ---
        When setting self.pk to None and then calling self.save(),
        Django will clone the communication along with all of its data
        and give it a new primary key. On the next iteration of the loop,
        the clone will be cloned along with its data, and so on. Same thing
        goes for each file attached to the communication.
        """
        # avoid circular imports
        from muckrock.foia.tasks import upload_document_cloud
        request_list = requests_from_pks(foia_pks)
        if not request_list:
            raise ValueError('No valid request(s) provided for cloning.')
        cloned_comms = []
        original_pk = self.pk
        files = self.files.all()
        for request in request_list:
            this_clone = FOIACommunication.objects.get(pk=original_pk)
            this_clone.pk = None
            this_clone.foia = request
            this_clone.save()
            for file_ in files:
                original_file_id = file_.id
                file_.pk = None
                file_.foia = request
                file_.comm = this_clone
                # make a copy of the file on the storage backend
                try:
                    new_ffile = ContentFile(file_.ffile.read())
                except ValueError:
                    error_msg = ('FOIAFile #%s has no data in its ffile field. '
                                'It has not been cloned.')
                    logging.error(error_msg, original_file_id)
                    continue
                new_ffile.name = file_.ffile.name
                file_.ffile = new_ffile
                file_.save()
                upload_document_cloud.apply_async(args=[file_.pk, False], countdown=3)
            # for each clone, self gets overwritten. each clone needs to be stored explicitly.
            cloned_comms.append(this_clone)
            logging.info('Communication #%d cloned to request #%d', original_pk, this_clone.foia.id)
        return cloned_comms

    def resend(self, email_address=None):
        """Resend the communication"""
        foia = self.foia
        if not foia:
            logging.error('Tried resending an orphaned communication.')
            raise ValueError('This communication has no FOIA to submit.')
        snail = False
        self.date = datetime.datetime.now()
        self.save()
        if email_address:
            # responsibility for handling validation errors
            # is on the caller of the resend method
            validate_email(email_address)
            foia.email = email_address
            foia.save()
        else:
            snail = True
        foia.submit(snail=snail)
        logging.info('Communication #%d resent.', self.id)

    def set_raw_email(self, msg):
        """Set the raw email for this communication"""
        raw_email = RawEmail.objects.get_or_create(communication=self)[0]
        raw_email.raw_email = msg
        raw_email.save()

    def get_sender_email(self):
        """Get the email this communication was sent from."""
        _, sender_email = email.utils.parseaddr(self.priv_from_who)
        try:
            validate_email(sender_email)
        except ValidationError:
            # if the sender email is invalid, don't return anything
            sender_email = None
        return sender_email

    def make_sender_primary_contact(self):
        """Makes the communication's sender the primary email of its FOIA."""
        sender_email = self.get_sender_email()
        if not sender_email:
            raise ValueError('Communication was not sent from a valid email.')
        if not self.foia:
            raise ValueError('Communication is an orphan and has no associated request.')
        self.foia.email = sender_email
        self.foia.save()
        return

    def _presave_special_handling(self):
        """Special handling before saving
        For example, strip out BoP excessive quoting"""

        def test_agency_name(name):
            """Match on agency name"""
            return (self.foia and self.foia.agency and
                    self.foia.agency.name == name)

        def until_string(string):
            """Cut communication off after string"""
            def modify():
                """Run the modification on self.communication"""
                if string in self.communication:
                    idx = self.communication.index(string)
                    self.communication = self.communication[:idx]
            return modify

        special_cases = [
            # BoP: strip everything after '>>>'
            (test_agency_name('Bureau of Prisons'),
             until_string('>>>')),
            # Phoneix Police: strip everything after '_'*32
            (test_agency_name('Phoenix Police Department'),
             until_string('_' * 32)),
        ]

        for test, modify in special_cases:
            if test:
                modify()

    class Meta:
        # pylint: disable=too-few-public-methods
        ordering = ['date']
        verbose_name = 'FOIA Communication'
        app_label = 'foia'

class RawEmail(models.Model):
    """The raw email text for a communication - stored seperately for performance"""

    communication = models.OneToOneField(FOIACommunication)
    raw_email = models.TextField(blank=True)

    def __unicode__(self):
        return 'Raw Email: %d' % self.pk

    class Meta:
        app_label = 'foia'

class FOIANote(models.Model):
    """A private note on a FOIA request"""

    foia = models.ForeignKey(FOIARequest, related_name='notes')
    author = models.ForeignKey('auth.User', related_name='notes', null=True)
    datetime = models.DateTimeField()
    note = models.TextField()

    def __unicode__(self):
        # pylint: disable=no-member
        if self.author:
            author = self.author
        else:
            author = self.foia.user
        return 'Note by %s on %s' % (author.get_full_name(), self.foia.title)

    class Meta:
        # pylint: disable=too-few-public-methods
        ordering = ['foia', 'datetime']
        verbose_name = 'FOIA Note'
        app_label = 'foia'
<|endoftext|>


<commit_before><commit_msg>download audio using raw API<commit_after><|endoftext|>


<commit_before><commit_msg>fix 'Incorrect signature: server authorization' error<commit_after><|endoftext|>


<commit_before><commit_msg>Add better default behavior for notebook plotting<commit_after><|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Vortex2D documentation build configuration file, created by
# sphinx-quickstart on Sun Nov 26 20:15:01 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
import subprocess
import pip

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

sys.path.append('.')

if on_rtd:
    pip.main(['install', 'sphinx_rtd_theme'])
    pip.main(['install', 'breathe'])

    subprocess.call('doxygen', shell=True)

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['breathe']

breathe_projects = {'Vortex2D' : 'xml_output/xml'}
breathe_default_project = 'Vortex2D'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Vortex2D'
copyright = u'2017, Maximilian Maldacker'
author = u'Maximilian Maldacker'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u'1.0'
# The full version, including alpha/beta/rc tags.
release = u'1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = []

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = "sphinx_rtd_theme"

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
    '**': [
        'relations.html',  # needs 'show_related': True theme option to display
        'searchbox.html',
    ]
}

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'Vortex2Ddoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #
    # 'preamble': '',

    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'Vortex2D.tex', u'Vortex2D Documentation',
     u'Maximilian Maldacker', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'vortex2d', u'Vortex2D Documentation',
     [author], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'Vortex2D', u'Vortex2D Documentation',
     author, 'Vortex2D', 'One line description of project.',
     'Miscellaneous'),
]

<commit_msg>fix sphinx config<commit_after># -*- coding: utf-8 -*-
#
# Vortex2D documentation build configuration file, created by
# sphinx-quickstart on Sun Nov 26 20:15:01 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
import subprocess
import pip

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

sys.path.append('.')

if on_rtd:
    subprocess.call('doxygen', shell=True)

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['breathe']

breathe_projects = {'Vortex2D' : 'xml_output/xml'}
breathe_default_project = 'Vortex2D'

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Vortex2D'
copyright = u'2017, Maximilian Maldacker'
author = u'Maximilian Maldacker'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = u'1.0'
# The full version, including alpha/beta/rc tags.
release = u'1.0'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = []

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = "sphinx_rtd_theme"

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
    '**': [
        'relations.html',  # needs 'show_related': True theme option to display
        'searchbox.html',
    ]
}

# -- Options for HTMLHelp output ------------------------------------------

# Output file base name for HTML help builder.
htmlhelp_basename = 'Vortex2Ddoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    #
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    #
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    #
    # 'preamble': '',

    # Latex figure (float) alignment
    #
    # 'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    (master_doc, 'Vortex2D.tex', u'Vortex2D Documentation',
     u'Maximilian Maldacker', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'vortex2d', u'Vortex2D Documentation',
     [author], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    (master_doc, 'Vortex2D', u'Vortex2D Documentation',
     author, 'Vortex2D', 'One line description of project.',
     'Miscellaneous'),
]

<|endoftext|>


<commit_before>from collections import defaultdict
from datetime import datetime
import logging
from pathlib import Path

from django.core.urlresolvers import reverse_lazy
import magic
from vanilla import TemplateView, FormView

from ..models import IconMapping
from ..forms import FileBrowserForm
from ..utils.zfs import ZFSUtil
from ..views.base import BaseView

logger = logging.getLogger(__name__)

class FileBrowser(BaseView, FormView):
    """
    Browse live filesystem using python os stdlib
    """
    template_name = 'file_list.html'
    headline = 'File Browser'
    form_class = FileBrowserForm
    success_url = reverse_lazy('nasman:file-browser')

    def get_context_data(self, **kwargs):
        """
        Build the context data for the file browser
        """
        context = super(FileBrowser, self).get_context_data(**kwargs)
        form = context['form']
        if form.is_bound:
            form.is_valid()
            # filesystem = form.cleaned_data.get('filesystem')
            # snapshot = form.cleaned_data.get('snapshot')
        if 'path' in form.changed_data:
            path = form.cleaned_data['path']
        elif 'filesystem' in form.changed_data:
            path = Path(form.cleaned_data['filesystem'].mountpoint)
        elif 'snapshot' in form.changed_data:
            snapshot = form.cleaned_data['snapshot']
            snapshot.mount()
            path = Path(snapshot.mountpoint)
        else:
            path = Path('/')
        root = path.root
        icon_mapping = defaultdict(
            lambda: 'fa-file-o',
            {x.mime_type: x.icon
             for x in IconMapping.objects.all()})

        # The file list/browser
        object_list = []
        for x in path.iterdir():
            try:
                mime_type = magic.from_file(str(x), mime=True).decode('utf8')
            except OSError as e:
                logger.warn('Unable to get mime_type of %s', str(x))
                mime_type = None
            object_list.append({
                'name': x.name,
                'full_path': x,
                'directory': x.is_dir(),
                'mime_type': mime_type,
                'modified': datetime.fromtimestamp(x.lstat().st_mtime),
                'size': x.lstat().st_size,
                'icon': icon_mapping[mime_type],
            })
        object_list.sort(key=lambda k: (not k['directory'], k['name']))

        # Path breadcrumbs
        breadcrumbs = sorted(path.parents) + [path]

        # The up one '..' link
        up_one = None
        if path != root:
            up_one = path.parent

        # Snapshot list for sidebar
        extra_sidebar = {
            'id': 'snapshot-list',
            'header': 'Snapshots',
            'list': [
                {'link': x.name,
                 'title': x.name} for x in ZFSUtil.get_snapshots()
            ]
        }
        context.update({
            'up_one': up_one,
            'object_list': object_list,
            'path': breadcrumbs,
            'extra_sidebar': extra_sidebar,
            # 'filesystem': filesystem,
            # 'snapshot': snapshot
        })
        return context

    def form_valid(self, form):
        return self.form_invalid(form)

class FilesystemList(BaseView, TemplateView):
    template_name = 'filesystem_list.html'
    headline = 'ZFS Filesystems'

    def get_context_data(self, **kwargs):
        context = super(FilesystemList, self).get_context_data()
        context['object_list'] = ZFSUtil.get_filesystems()
        return context
<commit_msg>Refactored path generation<commit_after>from collections import defaultdict
from datetime import datetime
import logging
from pathlib import Path

from django.core.urlresolvers import reverse_lazy
import magic
from vanilla import TemplateView, FormView

from ..models import IconMapping
from ..forms import FileBrowserForm
from ..utils.zfs import ZFSUtil
from ..views.base import BaseView

logger = logging.getLogger(__name__)

class FileBrowser(BaseView, FormView):
    """
    Browse live filesystem using python os stdlib
    """
    template_name = 'file_list.html'
    headline = 'File Browser'
    form_class = FileBrowserForm
    success_url = reverse_lazy('nasman:file-browser')

    def get_context_data(self, **kwargs):
        """
        Build the context data for the file browser
        """
        context = super(FileBrowser, self).get_context_data(**kwargs)
        form = context['form']
        filesystem = None
        snapshot = None
        if form.is_bound:
            form.is_valid()
            filesystem = form.cleaned_data.get('filesystem')
            snapshot = form.cleaned_data.get('snapshot')
        if 'path' in form.changed_data:
            path = form.cleaned_data['path']
        elif 'filesystem' in form.changed_data:
            filesystem.mount()
            path = Path(filesystem.mountpoint)
        elif 'snapshot' in form.changed_data:
            snapshot.mount()
            path = Path(snapshot.mountpoint)
        else:
            path = Path('/')
        root = path.root
        icon_mapping = defaultdict(
            lambda: 'fa-file-o',
            {x.mime_type: x.icon
             for x in IconMapping.objects.all()})

        # The file list/browser
        object_list = []
        for x in path.iterdir():
            try:
                mime_type = magic.from_file(str(x), mime=True).decode('utf8')
            except OSError as e:
                logger.warn('Unable to get mime_type of %s', str(x))
                mime_type = None
            object_list.append({
                'name': x.name,
                'full_path': x,
                'directory': x.is_dir(),
                'mime_type': mime_type,
                'modified': datetime.fromtimestamp(x.lstat().st_mtime),
                'size': x.lstat().st_size,
                'icon': icon_mapping[mime_type],
            })
        object_list.sort(key=lambda k: (not k['directory'], k['name']))

        # Path breadcrumbs
        breadcrumbs = sorted(path.parents) + [path]

        # The up one '..' link
        up_one = None
        if path != root:
            up_one = path.parent

        # Snapshot list for sidebar
        extra_sidebar = {
            'id': 'snapshot-list',
            'header': 'Snapshots',
            'list': [
                {'link': x.name,
                 'title': x.name} for x in ZFSUtil.get_snapshots()
            ]
        }
        context.update({
            'up_one': up_one,
            'object_list': object_list,
            'path': breadcrumbs,
            'extra_sidebar': extra_sidebar,
            # 'filesystem': filesystem,
            # 'snapshot': snapshot
        })
        return context

    def form_valid(self, form):
        return self.form_invalid(form)

class FilesystemList(BaseView, TemplateView):
    template_name = 'filesystem_list.html'
    headline = 'ZFS Filesystems'

    def get_context_data(self, **kwargs):
        context = super(FilesystemList, self).get_context_data()
        context['object_list'] = ZFSUtil.get_filesystems()
        return context
<|endoftext|>


<commit_before># Python main script
import random
from subprocess import Popen
import os.path, time, glob, shutil, mysql.connector, ConfigParser
import subprocess
import sys
import signal

# import workers
from db import DB
import hyperneat_worker as hn
import voxelyze_worker as vox
import postprocessing_worker as pp

class EC14controller():
    base_path = ""
    db = None
    pause_time = 10
    time_start = time.time()
    timer = 0
    wall_time = 290
    run = 0
    dbString = ""
    dbParams = ""
    exp_name = ""
    hnWorker = None
    voxWorker = None
    ppWorker = None
    newExperiment = False
    pop_size = 0
    pop_random = 0
    pop_random_start_end = (0, 0)
    indiv_max_age = 0
    arena_x = 0
    arena_y = 0
    arena_type = ""
    config = None
    path_prefix = "~/EC14-Exp-"
    mailer = False
    mailer_subject = "Experiment {exp_name} done"
    mailer_content = "Experiment {exp_name} successfully completed. Total population: {pop_total}"

    random_granularity = 10000.0

    yes = {'yes', 'y', 'ye'}
    no = {'no', 'n', ''}

    def __init__(self):
        self.config = ConfigParser.RawConfigParser()

    def installFiles(self):
        """ Copy script files into experiment directory and saves config
        :return: None
        """

        os.makedirs(self.base_path)
        os.makedirs(self.base_path + "scripts/")
        os.makedirs(self.base_path + "config/")
        os.makedirs(self.base_path + "population/")
        os.makedirs(self.base_path + "traces_duringVox/")
        os.makedirs(self.base_path + "traces_afterVox/")
        os.makedirs(self.base_path + "pool/")
        os.makedirs(self.base_path + "logs/")

        shutil.copy(self.configPath, self.base_path + "config/config.ini")

        files = glob.iglob("./*.py")
        for f in files:
            if os.path.isfile(f):
                shutil.copy(f, self.base_path + "scripts/")

    def getDB(self):
        """ retrieve database string from config or cache
        :return: None
        """

        self.db = DB(self.dbString, self.exp_name, self.end_time, self.indiv_max_age)
        self.dbParams = (self.dbString, self.exp_name, self.end_time, self.indiv_max_age)

    def createPopulaton(self):
        """ creates the initial population in the database
        :return: None
        """

        for i in range(self.pop_size):
            birth = 0
            if (self.pop_random):
                birth = (random.randrange(self.pop_random_start * self.random_granularity,
                                          self.pop_random_end * self.random_granularity)
                         / self.random_granularity)
            x = random.randrange(0, self.arena_x * self.random_granularity) / self.random_granularity
            y = random.randrange(0, self.arena_y * self.random_granularity) / self.random_granularity
            self.db.createIndividual(birth, x, y)

    def readConfig(self, filename):
        self.config.read(filename)
        self.dbString = self.config.get('DB', 'db_string')

        self.exp_name = self.config.get('Experiment', 'name')
        self.path_prefix = self.config.get('Experiment', 'path_prefix')
        self.debug = self.config.getboolean('Experiment', 'debug')
        self.wall_time = self.config.getint('Experiment', 'self_wall_time')
        self.end_time = self.config.getfloat('Experiment', 'end_time')
        self.random_granularity = self.config.getfloat('Experiment', 'random_granularity')

        self.arena_y = self.config.getfloat('Arena', 'x')
        self.arena_x = self.config.getfloat('Arena', 'y')
        self.arena_type = self.config.get('Arena', 'type')

        self.pop_size = self.config.getint('Population', 'size')
        self.pop_random = self.config.getboolean('Population', 'random')
        self.pop_random_start = self.config.getfloat('Population', 'random_start')
        self.pop_random_end = self.config.getfloat('Population', 'random_end')

        self.pause_time = self.config.getint('Workers', 'pause_time')

        self.mailer = self.config.getboolean('Mailer', 'mailer')
        self.mailer_content = self.config.get('Mailer', 'content')
        self.mailer_subject = self.config.get('Mailer', 'subject')

    def isNewExperiment(self):
        self.base_path = os.path.expanduser(self.path_prefix + self.exp_name) + "/"
        print(self.base_path)
        if not os.path.isdir(self.base_path):
            return True
        return False

    def initialize(self):
        self.readConfig(self.configPath)
        self.getDB()
        if (self.isNewExperiment()):
            self.installFiles()
            self.db.dropTables()
            self.db.createTables()
            self.createPopulaton()

    def launchWorkers(self):
        # launch workers
        self.hnWorker = hn.HNWorker(self.dbParams, self.configPath)
        self.hnWorker.start()
        self.voxWorker = vox.VoxWorker(self.dbParams, self.configPath)
        self.voxWorker.start()
        self.ppWorker = pp.PostprocessingWorker(self.dbParams, self.configPath)
        self.ppWorker.start()

    def handleParams(self):
        if len(sys.argv) > 3:
            print(
                "I take a maximum of 2 arguments: first the relative path to the config file, then [optional] the number of the resubmit... quitting.")
            quit()

        if len(sys.argv) == 3:
            self.run = int(sys.argv[2])

        if len(sys.argv) > 1:
            self.configPath = sys.argv[1]
        else:
            self.configPath = "./config.ini"

        self.configPath = os.path.abspath(self.configPath)
        print("using config file: " + self.configPath)

    def start(self):
        self.handleParams()
        self.initialize()
        self.launchWorkers()

        signal.signal(signal.SIGINT, self.keyboard_exit)

        while (time.time() - self.time_start <= self.wall_time - self.pause_time):
            unfinished = self.db.getUnfinishedIndividuals()
            if unfinished == 0:
                print("nothing left to do, quiting")
                if (self.mailer):
                    self.sendFinishedMail()
                self.clean_exit()
            time.sleep(self.pause_time)

        self.resubmit()

    def resubmit(self):
        logPrefix = "main.run{n}".format(n=self.run + 1)
        logPath = self.base_path + "logs/" + logPrefix
        cwd = os.path.dirname(os.path.realpath(__file__))

        cmd = "qsub -o {logpath}.output.log -e {logpath}.error.log -l walltime={walltime} -v "+\
              "config={config},run={run},cwd={cwd} {cwd}/scripts/main-resub.sh"

        qsub = subprocess.check_output(cmd.format(logpath=logPath,
                                                  walltime=self.wall_time,
                                                  config=self.configPath,
                                                  run=self.run + 1,
                                                  cwd=cwd),
                                       shell=True)
        print("MAIN: resubmitted myself as:" + str(qsub) )

    def keyboard_exit(self, signal, frame):
        print("\n-------------\nreceived CTRL+C... exiting gracefully.\n-------------\n")
        self.clean_exit()

    def clean_exit(self):
        self.hnWorker.join()
        self.voxWorker.join()
        self.ppWorker.join()
        sys.exit(0)

    def sendFinishedMail(self):
        pop_total = self.db.getPopulationTotal()
        subject = self.mailer_subject.format(exp_name = self.exp_name, pop_total = pop_total)
        content = self.mailer_subject.format(exp_name = self.exp_name, pop_total = pop_total)
        mailer_cmd = "echo '{content}' | mail $USER -s '{subject}'".format(content = content, subject = subject)
        subprocess.Popen(mailer_cmd, shell=True)

ctrl = EC14controller()
ctrl.start()
<commit_msg>damn you copy&paste!<commit_after># Python main script
import random
from subprocess import Popen
import os.path, time, glob, shutil, mysql.connector, ConfigParser
import subprocess
import sys
import signal

# import workers
from db import DB
import hyperneat_worker as hn
import voxelyze_worker as vox
import postprocessing_worker as pp

class EC14controller():
    base_path = ""
    db = None
    pause_time = 10
    time_start = time.time()
    timer = 0
    wall_time = 290
    run = 0
    dbString = ""
    dbParams = ""
    exp_name = ""
    hnWorker = None
    voxWorker = None
    ppWorker = None
    newExperiment = False
    pop_size = 0
    pop_random = 0
    pop_random_start_end = (0, 0)
    indiv_max_age = 0
    arena_x = 0
    arena_y = 0
    arena_type = ""
    config = None
    path_prefix = "~/EC14-Exp-"
    mailer = False
    mailer_subject = "Experiment {exp_name} done"
    mailer_content = "Experiment {exp_name} successfully completed. Total population: {pop_total}"

    random_granularity = 10000.0

    yes = {'yes', 'y', 'ye'}
    no = {'no', 'n', ''}

    def __init__(self):
        self.config = ConfigParser.RawConfigParser()

    def installFiles(self):
        """ Copy script files into experiment directory and saves config
        :return: None
        """

        os.makedirs(self.base_path)
        os.makedirs(self.base_path + "scripts/")
        os.makedirs(self.base_path + "config/")
        os.makedirs(self.base_path + "population/")
        os.makedirs(self.base_path + "traces_duringVox/")
        os.makedirs(self.base_path + "traces_afterVox/")
        os.makedirs(self.base_path + "pool/")
        os.makedirs(self.base_path + "logs/")

        shutil.copy(self.configPath, self.base_path + "config/config.ini")

        files = glob.iglob("./*.py")
        for f in files:
            if os.path.isfile(f):
                shutil.copy(f, self.base_path + "scripts/")

    def getDB(self):
        """ retrieve database string from config or cache
        :return: None
        """

        self.db = DB(self.dbString, self.exp_name, self.end_time, self.indiv_max_age)
        self.dbParams = (self.dbString, self.exp_name, self.end_time, self.indiv_max_age)

    def createPopulaton(self):
        """ creates the initial population in the database
        :return: None
        """

        for i in range(self.pop_size):
            birth = 0
            if (self.pop_random):
                birth = (random.randrange(self.pop_random_start * self.random_granularity,
                                          self.pop_random_end * self.random_granularity)
                         / self.random_granularity)
            x = random.randrange(0, self.arena_x * self.random_granularity) / self.random_granularity
            y = random.randrange(0, self.arena_y * self.random_granularity) / self.random_granularity
            self.db.createIndividual(birth, x, y)

    def readConfig(self, filename):
        self.config.read(filename)
        self.dbString = self.config.get('DB', 'db_string')

        self.exp_name = self.config.get('Experiment', 'name')
        self.path_prefix = self.config.get('Experiment', 'path_prefix')
        self.debug = self.config.getboolean('Experiment', 'debug')
        self.wall_time = self.config.getint('Experiment', 'self_wall_time')
        self.end_time = self.config.getfloat('Experiment', 'end_time')
        self.random_granularity = self.config.getfloat('Experiment', 'random_granularity')

        self.arena_y = self.config.getfloat('Arena', 'x')
        self.arena_x = self.config.getfloat('Arena', 'y')
        self.arena_type = self.config.get('Arena', 'type')

        self.pop_size = self.config.getint('Population', 'size')
        self.pop_random = self.config.getboolean('Population', 'random')
        self.pop_random_start = self.config.getfloat('Population', 'random_start')
        self.pop_random_end = self.config.getfloat('Population', 'random_end')

        self.pause_time = self.config.getint('Workers', 'pause_time')

        self.mailer = self.config.getboolean('Mailer', 'mailer')
        self.mailer_content = self.config.get('Mailer', 'content')
        self.mailer_subject = self.config.get('Mailer', 'subject')

    def isNewExperiment(self):
        self.base_path = os.path.expanduser(self.path_prefix + self.exp_name) + "/"
        print(self.base_path)
        if not os.path.isdir(self.base_path):
            return True
        return False

    def initialize(self):
        self.readConfig(self.configPath)
        self.getDB()
        if (self.isNewExperiment()):
            self.installFiles()
            self.db.dropTables()
            self.db.createTables()
            self.createPopulaton()

    def launchWorkers(self):
        # launch workers
        self.hnWorker = hn.HNWorker(self.dbParams, self.configPath)
        self.hnWorker.start()
        self.voxWorker = vox.VoxWorker(self.dbParams, self.configPath)
        self.voxWorker.start()
        self.ppWorker = pp.PostprocessingWorker(self.dbParams, self.configPath)
        self.ppWorker.start()

    def handleParams(self):
        if len(sys.argv) > 3:
            print(
                "I take a maximum of 2 arguments: first the relative path to the config file, then [optional] the number of the resubmit... quitting.")
            quit()

        if len(sys.argv) == 3:
            self.run = int(sys.argv[2])

        if len(sys.argv) > 1:
            self.configPath = sys.argv[1]
        else:
            self.configPath = "./config.ini"

        self.configPath = os.path.abspath(self.configPath)
        print("using config file: " + self.configPath)

    def start(self):
        self.handleParams()
        self.initialize()
        self.launchWorkers()

        signal.signal(signal.SIGINT, self.keyboard_exit)

        while (time.time() - self.time_start <= self.wall_time - self.pause_time):
            unfinished = self.db.getUnfinishedIndividuals()
            if unfinished == 0:
                print("nothing left to do, quiting")
                if (self.mailer):
                    self.sendFinishedMail()
                self.clean_exit()
            time.sleep(self.pause_time)

        self.resubmit()

    def resubmit(self):
        logPrefix = "main.run{n}".format(n=self.run + 1)
        logPath = self.base_path + "logs/" + logPrefix
        cwd = os.path.dirname(os.path.realpath(__file__))

        cmd = "qsub -o {logpath}.output.log -e {logpath}.error.log -l walltime={walltime} -v "+\
              "config={config},run={run},cwd={cwd} {cwd}/scripts/main-resub.sh"

        qsub = subprocess.check_output(cmd.format(logpath=logPath,
                                                  walltime=self.wall_time,
                                                  config=self.configPath,
                                                  run=self.run + 1,
                                                  cwd=cwd),
                                       shell=True)
        print("MAIN: resubmitted myself as:" + str(qsub) )

    def keyboard_exit(self, signal, frame):
        print("\n-------------\nreceived CTRL+C... exiting gracefully.\n-------------\n")
        self.clean_exit()

    def clean_exit(self):
        self.hnWorker.join()
        self.voxWorker.join()
        self.ppWorker.join()
        sys.exit(0)

    def sendFinishedMail(self):
        pop_total = self.db.getPopulationTotal()
        subject = self.mailer_subject.format(exp_name = self.exp_name, pop_total = pop_total)
        content = self.mailer_content.format(exp_name = self.exp_name, pop_total = pop_total)
        mailer_cmd = "echo '{content}' | mail $USER -s '{subject}'".format(content = content, subject = subject)
        subprocess.Popen(mailer_cmd, shell=True)

ctrl = EC14controller()
ctrl.start()
<|endoftext|>


<commit_before><commit_msg>fix jsay.py を使うように修正<commit_after><|endoftext|>


<commit_before>from .parsers import WebVTTParser, SRTParser

SUPPORTED_FORMATS = (
    ('webvtt', WebVTTParser),  # default parser for WebVTT format
    ('srt',    SRTParser),     # parser for SRT format
)

class WebVTT:
    """
    Parse captions in WebVTT format and also from other formats like SRT.
    To read WebVTT:
        WebVTT().read('captions.vtt')
    For other formats like SRT, use from_[format in lower case]:
        WebVTT().from_srt('captions.srt')

    A list of all supported formats is available calling supported_formats().
    """
    def __init__(self):
        self.parser = None

        # create methods dynamically to read captions based on the supported types
        # read() is created for WebVTT and from_[FORMAT]() for the other formats.
        for format_, parser_class in SUPPORTED_FORMATS:
            method_name = 'read' if format_ == 'webvtt' else 'from_{}'.format(format_)

            setattr(self.__class__, method_name, self._set_reader(method_name, format_, parser_class))

    def _set_reader(self, name, format_, parser_class):
        def f(self, file):
            self.parser = parser_class().read(file)
            return self

        f.__name__ = name
        if format_ == 'webvtt':
            f.__doc__ = 'Reads a WebVTT captions file.'
        else:
            f.__doc__ = 'Reads captions from a file in {} format.'.format(format_.upper())
        return f

    @staticmethod
    def supported_formats():
        """Provides a list of supported formats that this class can read from."""
        return [f[0] for f in SUPPORTED_FORMATS]

    @property
    def captions(self):
        """Returns the list of parsed captions."""
        return [] if self.parser is None else self.parser.captions

    @property
    def total_length(self):
        """Returns the total length of the captions."""
        if not self.captions:
            return 0
        return int(self.captions[-1].end) - int(self.captions[0].start)<commit_msg>Remove parser reference and add save functionality<commit_after>import os
from .parsers import WebVTTParser, SRTParser

SUPPORTED_FORMATS = (
    ('webvtt', WebVTTParser),  # default parser for WebVTT format
    ('srt',    SRTParser),     # parser for SRT format
)

class WebVTT(object):
    """
    Parse captions in WebVTT format and also from other formats like SRT.
    To read WebVTT:
        WebVTT().read('captions.vtt')
    For other formats like SRT, use from_[format in lower case]:
        WebVTT().from_srt('captions.srt')

    A list of all supported formats is available calling supported_formats().
    """
    def __init__(self):
        self.captions = []
        self.file = ''

        # create methods dynamically to read captions based on the supported types
        # read() is created for WebVTT and from_[FORMAT]() for the other formats.
        for format_, parser_class in SUPPORTED_FORMATS:
            method_name = 'read' if format_ == 'webvtt' else 'from_{}'.format(format_)

            setattr(self.__class__, method_name, self._set_reader(method_name, format_, parser_class))

    def _set_reader(self, name, format_, parser_class):
        def f(self, file):
            self.file = file
            self.captions = parser_class().read(file).captions
            return self

        f.__name__ = name
        if format_ == 'webvtt':
            f.__doc__ = 'Reads a WebVTT captions file.'
        else:
            f.__doc__ = 'Reads captions from a file in {} format.'.format(format_.upper())
        return f

    def save(self):
        # saving an original vtt file will overwrite the file
        # and for files read from other formats will save as vtt
        # with the same name and location
        self.file = os.path.splitext(self.file)[0] + '.vtt'

        with open(self.file, 'w', encoding='utf-8') as f:
            f.write('WEBVTT\n')
            for c in self.captions:
                f.write('\n{} --> {}\n'.format(c.start_as_timestamp, c.end_as_timestamp))
                f.writelines(['{}\n'.format(l) for l in c.lines])

    @staticmethod
    def supported_formats():
        """Provides a list of supported formats that this class can read from."""
        return [f[0] for f in SUPPORTED_FORMATS]

    @property
    def total_length(self):
        """Returns the total length of the captions."""
        if not self.captions:
            return 0
        return int(self.captions[-1].end) - int(self.captions[0].start)<|endoftext|>


<commit_before># The MIT License (MIT)
# Copyright (c) 2014-2017 University of Bristol
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.
"""
Node module.

Nodes are a collection of streams defined by shared meta-data keys (plates), and are connected in the
computational graph by factors.
"""
from ..stream import StreamId
from ..utils import Printable

import logging

class Node(Printable):
    """
    A node in the graph. This consists of a set of streams defined over a set of plates
    """
    def __init__(self, node_id, streams, plate_ids, plate_values):
        """
        Initialise the node
        :param node_id: The node id
        :param streams: The streams, organised as a nested dictionary with plate objects as keys at the top level,
        and then plate values (tuple(sorted(plate_values.items())) as the keys at the next level
        :param plate_ids: The plate ids
        :param plate_values: The plate values (populated meta data)
        """
        self.node_id = node_id
        self.streams = streams
        # TODO: Remove plate IDs
        self.plate_ids = tuple(plate_ids) if plate_ids else tuple()
        self.plate_values = plate_values

        """
        When defining streams, it will be useful to be able to query node objects
        to determine the streams that have metadata of a particular value.
        Use Node.reverse_lookup as follows:
            meta_data = {'a': 1, 'b': 1}

        """

    def intersection(self, meta):
        """
        Get the intersection between the meta data given and the meta data contained within the plates.
        Since all of the streams have the same meta data keys (but differing values) we only need to consider
        the first stream.
        :param meta: The meta data to compare
        :return: A stream id with the intersection between this node's meta data and the given meta data
        :type meta: dict
        :rtype: StreamId
        """
        keys = self.streams[0].stream_id.meta_data.keys()
        return StreamId(self.node_id, dict(*zip((kk, meta[kk]) for kk in keys)))

    def print_head(self, parent_plate_value, plate_values, interval, n=10, print_func=logging.info):
        """
        Print the first n values from the streams in the given time interval.
        The parent plate value is the value of the parent plate,
        and then the plate values are the values for the plate that are to be printed.
         e.g. print_head()
        :param parent_plate_value: The (fixed) parent plate value
        :param plate_values: The plate values over which to loop
        :param interval: The time interval
        :param n: The maximum number of elements to print
        :param print_func: The function used for printing (e.g. logging.info() or print())
        :return: None
        """
        # Deferred import to avoid circular dependence
        from . import Plate
        if isinstance(plate_values, Plate):
            self.print_head(parent_plate_value, plate_values.values, interval, n, print_func)
            return

        if len(plate_values) == 1 and len(plate_values[0]) == 2 and isinstance(plate_values[0][0], str):
            self.print_head(parent_plate_value, (plate_values,), interval, n, print_func)
            return

        found = False
        for plate_value in plate_values:
            if parent_plate_value:
                plate_value = parent_plate_value + (plate_value,)

            if plate_value not in self.streams:
                # This can happen if we have created a compound plate and only certain plate values are valid
                continue

            found = True
            print_func("Plate value: {}".format(plate_value))
            data = False
            for k, v in self.streams[tuple(sorted(plate_value))].window(interval).head(n):
                data = True
                print_func("{}, {}".format(k, v))
            if not data:
                print_func("No data")
            print_func("")
        if not found:
            print_func("No streams found for the given plate values")
<commit_msg>Fixed node printing bug<commit_after># The MIT License (MIT)
# Copyright (c) 2014-2017 University of Bristol
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
# OR OTHER DEALINGS IN THE SOFTWARE.
"""
Node module.

Nodes are a collection of streams defined by shared meta-data keys (plates), and are connected in the
computational graph by factors.
"""
from ..stream import StreamId
from ..utils import Printable

import logging

class Node(Printable):
    """
    A node in the graph. This consists of a set of streams defined over a set of plates
    """
    def __init__(self, node_id, streams, plate_ids, plate_values):
        """
        Initialise the node
        :param node_id: The node id
        :param streams: The streams, organised as a nested dictionary with plate objects as keys at the top level,
        and then plate values (tuple(sorted(plate_values.items())) as the keys at the next level
        :param plate_ids: The plate ids
        :param plate_values: The plate values (populated meta data)
        """
        self.node_id = node_id
        self.streams = streams
        # TODO: Remove plate IDs
        self.plate_ids = tuple(plate_ids) if plate_ids else tuple()
        self.plate_values = plate_values

        """
        When defining streams, it will be useful to be able to query node objects
        to determine the streams that have metadata of a particular value.
        Use Node.reverse_lookup as follows:
            meta_data = {'a': 1, 'b': 1}

        """

    def intersection(self, meta):
        """
        Get the intersection between the meta data given and the meta data contained within the plates.
        Since all of the streams have the same meta data keys (but differing values) we only need to consider
        the first stream.
        :param meta: The meta data to compare
        :return: A stream id with the intersection between this node's meta data and the given meta data
        :type meta: dict
        :rtype: StreamId
        """
        keys = self.streams[0].stream_id.meta_data.keys()
        return StreamId(self.node_id, dict(*zip((kk, meta[kk]) for kk in keys)))

    def print_head(self, parent_plate_value, plate_values, interval, n=10, print_func=logging.info):
        """
        Print the first n values from the streams in the given time interval.
        The parent plate value is the value of the parent plate,
        and then the plate values are the values for the plate that are to be printed.
         e.g. print_head()
        :param parent_plate_value: The (fixed) parent plate value
        :param plate_values: The plate values over which to loop
        :param interval: The time interval
        :param n: The maximum number of elements to print
        :param print_func: The function used for printing (e.g. logging.info() or print())
        :return: None
        """
        # Deferred import to avoid circular dependence
        from . import Plate
        if isinstance(plate_values, Plate):
            self.print_head(parent_plate_value, plate_values.values, interval, n, print_func)
            return

        if len(plate_values) == 1 and len(plate_values[0]) == 2 and isinstance(plate_values[0][0], str):
            self.print_head(parent_plate_value, (plate_values,), interval, n, print_func)
            return

        found = False
        for plate_value in plate_values:
            if parent_plate_value:
                plate_value = parent_plate_value + (plate_value,)

            plate_value = tuple(sorted(plate_value))

            if plate_value not in self.streams:
                # This can happen if we have created a compound plate and only certain plate values are valid
                continue

            found = True
            print_func("Plate value: {}".format(plate_value))
            data = False
            for k, v in self.streams[plate_value].window(interval).head(n):
                data = True
                print_func("{}, {}".format(k, v))
            if not data:
                print_func("No data")
            print_func("")
        if not found:
            print_func("No streams found for the given plate values")
<|endoftext|>