<commit_before># -*- coding: utf-8 -*-

"""Pygments lexer for SISAL.

SISAL stands for Streams and Iteration In A Single-Assignment Language.

"""

from pygments.lexer import RegexLexer, Text, words, bygroups, include, default
from pygments.token import Comment, Name, Keyword, Number, Operator, String

__version__ = '0.1.1'
__author__ = 'Alexander Asp Bock'
__license__ = 'MIT'

class SisalLexer(RegexLexer):
    """Lexer for the SISAL programming language."""

    name = 'sisal'
    aliases = ['sis', 'sisal']
    filenames = ['*.sis', '*.sisal']
    tokens = {
        'root': [
            (r'\n', Text),
            (r'\\\n', Text),
            (r'\\', Text),
            (r'(define)(\s+)(.+)', bygroups(Comment.Preproc, Text, Text)),
            (r'(global)(\s+)(.+)', bygroups(Comment.Preproc, Text, Text)),
            (r'^%\$', Comment.Directive, 'directive'),
            (r'% .+$', Comment.Singleline),
            (r'\'', String.Single, 'singlequoted_string'),
            (r'"', String.Double, 'doublequoted_string'),
            (r'(function)(\s+)(.+)(\()',
             bygroups(Keyword, Text, Name.Function, Text)),
            (r'error', Name.Exception),
            (r'\(|\)', Text),
            include('keywords'),
            include('conversions'),
            include('types'),
            include('builtins'),
            include('numbers'),
            include('operators'),
            include('conditionals'),
            include('loops'),
            (r'[^\S\n]+', Text),
            (r'\w+', Text)
        ],
        'keywords': [
            (words(['sum',
                    'product',
                    'greatest',
                    'least',
                    'catenate',
                    'let',
                    'is',
                    'tagcase',
                    'otherwise',
                    'type',
                    'returns',
                    'forward',
                    # 'define',
                    'old',
                    'of',
                    'nil',
                    'in',
                    'end let',
                    'end function'],
                   prefix=r'\b',
                   suffix=r'\b'),
             Keyword.Reserved)
        ],
        'types': [
            (words(['integer',
                    'real',
                    'double_real',
                    'character',
                    'null',
                    'boolean',
                    'array',
                    'stream',
                    'record',
                    'union'],
                   suffix=r'\b'),
             Keyword.Type)
        ],
        'builtins': [
            (words(['abs',  # Standard functions
                    'exp',
                    'mod',
                    'floor',
                    'trunc',
                    'max',
                    'min',
                    # Array functions
                    'array_fill',
                    'array_limh',
                    'array_liml',
                    'array_size',
                    'array_prefixsize',
                    'array_adjust',
                    'array_addh',
                    'array_addl',
                    'array_remh',
                    'array_reml',
                    'array_setl',
                    # Stream functions
                    'stream_append',
                    'stream_first',
                    'stream_rest',
                    'stream_empty',
                    'stream_size',
                    'stream_prefixsize',
                    # Record functions
                    'replace'],
                   suffix=r'\b'),
             Name.Builtin),
        ],
        'conversions': [
            (r'(integer)(\()',     bygroups(Keyword.Reserved, Text)),
            (r'(real)(\()',        bygroups(Keyword.Reserved, Text)),
            (r'(double_real)(\()', bygroups(Keyword.Reserved, Text)),
            (r'(character)(\()',   bygroups(Keyword.Reserved, Text))
        ],
        'numbers': [
            (r'-?\d+\.\d+([eEdD][+-]?\d+)?', Number.Float),
            (r'-?\d+',                       Number.Integer)
        ],
        'operators': [
            ('\*',   Operator),
            ('\/',   Operator),
            ('\+',   Operator),
            ('-',    Operator),
            ('\|\|', Operator),
            ('~',    Operator),
            ('&',    Operator),
            ('=',    Operator),
            ('~=',   Operator),
            ('<',    Operator),
            ('>',    Operator),
            ('<=',   Operator),
            ('>=',   Operator),
            (':=',   Operator),
            (':',    Operator),
            ('\[',   Operator),
            ('\]',   Operator),
            (',',    Operator),
            (';',    Operator)
        ],
        'conditionals': [
            (words(['if',
                    'then',
                    'elseif',
                    'else',
                    'end if'],
                   suffix=r'\b'),
             Keyword)
        ],
        'loops': [
            (words(['for',
                    'while',
                    'initial',
                    'repeat',
                    'until',
                    'when',
                    'dot',
                    'cross',
                    'left',
                    'right',
                    'tree',
                    'end for'],
                   suffix=r'\b'),
             Keyword)
        ],
        'singlequoted_string': [
            (r'[^\'\\]+', String.Single),
            (r'\\.',       String.Escape),
            (r'\'',        String.Single, '#pop')
        ],
        'doublequoted_string': [
            (r'[^"\\]+', String.Double),
            (r'\\.',      String.Escape),
            (r'"',        String.Double, '#pop')
        ],
        'directive': [
            (words(['INCLUDE', 'SUBRANGE', 'PACKED', 'MAIN']),
             Comment.PreProc),
            default('#pop')
        ]}
<commit_msg>Added missing boolean OR operator<commit_after># -*- coding: utf-8 -*-

"""Pygments lexer for SISAL.

SISAL stands for Streams and Iteration In A Single-Assignment Language.

"""

from pygments.lexer import RegexLexer, Text, words, bygroups, include, default
from pygments.token import Comment, Name, Keyword, Number, Operator, String

__version__ = '0.1.1'
__author__ = 'Alexander Asp Bock'
__license__ = 'MIT'

class SisalLexer(RegexLexer):
    """Lexer for the SISAL programming language."""

    name = 'sisal'
    aliases = ['sis', 'sisal']
    filenames = ['*.sis', '*.sisal']
    tokens = {
        'root': [
            (r'\n', Text),
            (r'\\\n', Text),
            (r'\\', Text),
            (r'(define)(\s+)(.+)', bygroups(Comment.Preproc, Text, Text)),
            (r'(global)(\s+)(.+)', bygroups(Comment.Preproc, Text, Text)),
            (r'^%\$', Comment.Directive, 'directive'),
            (r'% .+$', Comment.Singleline),
            (r'\'', String.Single, 'singlequoted_string'),
            (r'"', String.Double, 'doublequoted_string'),
            (r'(function)(\s+)(.+)(\()',
             bygroups(Keyword, Text, Name.Function, Text)),
            (r'error', Name.Exception),
            (r'\(|\)', Text),
            include('keywords'),
            include('conversions'),
            include('types'),
            include('builtins'),
            include('numbers'),
            include('operators'),
            include('conditionals'),
            include('loops'),
            (r'[^\S\n]+', Text),
            (r'\w+', Text)
        ],
        'keywords': [
            (words(['sum',
                    'product',
                    'greatest',
                    'least',
                    'catenate',
                    'let',
                    'is',
                    'tagcase',
                    'otherwise',
                    'type',
                    'returns',
                    'forward',
                    # 'define',
                    'old',
                    'of',
                    'nil',
                    'in',
                    'end let',
                    'end function'],
                   prefix=r'\b',
                   suffix=r'\b'),
             Keyword.Reserved)
        ],
        'types': [
            (words(['integer',
                    'real',
                    'double_real',
                    'character',
                    'null',
                    'boolean',
                    'array',
                    'stream',
                    'record',
                    'union'],
                   suffix=r'\b'),
             Keyword.Type)
        ],
        'builtins': [
            (words(['abs',  # Standard functions
                    'exp',
                    'mod',
                    'floor',
                    'trunc',
                    'max',
                    'min',
                    # Array functions
                    'array_fill',
                    'array_limh',
                    'array_liml',
                    'array_size',
                    'array_prefixsize',
                    'array_adjust',
                    'array_addh',
                    'array_addl',
                    'array_remh',
                    'array_reml',
                    'array_setl',
                    # Stream functions
                    'stream_append',
                    'stream_first',
                    'stream_rest',
                    'stream_empty',
                    'stream_size',
                    'stream_prefixsize',
                    # Record functions
                    'replace'],
                   suffix=r'\b'),
             Name.Builtin),
        ],
        'conversions': [
            (r'(integer)(\()',     bygroups(Keyword.Reserved, Text)),
            (r'(real)(\()',        bygroups(Keyword.Reserved, Text)),
            (r'(double_real)(\()', bygroups(Keyword.Reserved, Text)),
            (r'(character)(\()',   bygroups(Keyword.Reserved, Text))
        ],
        'numbers': [
            (r'-?\d+\.\d+([eEdD][+-]?\d+)?', Number.Float),
            (r'-?\d+',                       Number.Integer)
        ],
        'operators': [
            ('\*',   Operator),
            ('\/',   Operator),
            ('\+',   Operator),
            ('-',    Operator),
            ('\|\|', Operator),
            ('\|',   Operator),
            ('~',    Operator),
            ('&',    Operator),
            ('=',    Operator),
            ('~=',   Operator),
            ('<',    Operator),
            ('>',    Operator),
            ('<=',   Operator),
            ('>=',   Operator),
            (':=',   Operator),
            (':',    Operator),
            ('\[',   Operator),
            ('\]',   Operator),
            (',',    Operator),
            (';',    Operator)
        ],
        'conditionals': [
            (words(['if',
                    'then',
                    'elseif',
                    'else',
                    'end if'],
                   suffix=r'\b'),
             Keyword)
        ],
        'loops': [
            (words(['for',
                    'while',
                    'initial',
                    'repeat',
                    'until',
                    'when',
                    'dot',
                    'cross',
                    'left',
                    'right',
                    'tree',
                    'end for'],
                   suffix=r'\b'),
             Keyword)
        ],
        'singlequoted_string': [
            (r'[^\'\\]+', String.Single),
            (r'\\.',       String.Escape),
            (r'\'',        String.Single, '#pop')
        ],
        'doublequoted_string': [
            (r'[^"\\]+', String.Double),
            (r'\\.',      String.Escape),
            (r'"',        String.Double, '#pop')
        ],
        'directive': [
            (words(['INCLUDE', 'SUBRANGE', 'PACKED', 'MAIN']),
             Comment.PreProc),
            default('#pop')
        ]}
<|endoftext|>


<commit_before># coding=utf-8

import pygame
import pygame.locals

class Board(object):
    """
    Plansza do gry. Odpowiada za rysowanie okna gry.
    """

    def __init__(self, width, height):
        """
        Konstruktor planszy do gry. Przygotowuje okienko gry.

        :param width:
        :param height:
        """
        self.surface = pygame.display.set_mode((width, height), 0, 32)
        pygame.display.set_caption('Simple Pong')

    def draw(self, *args):
        """
        Rysuje okno gry

        :param args: lista obiektów do narysowania
        """
        background = (230, 255, 255)
        self.surface.fill(background)
        for drawable in args:
            drawable.draw_on(self.surface)

        # dopiero w tym miejscu następuje fatyczne rysowanie
        # w oknie gry, wcześniej tylko ustalaliśmy co i jak ma zostać narysowane
        pygame.display.update()

class PongGame(object):
    """
    Łączy wszystkie elementy gry w całość.
    """

    def __init__(self, width, height):
        pygame.init()
        self.board = Board(width, height)
        # zegar którego użyjemy do kontrolowania szybkości rysowania
        # kolejnych klatek gry
        self.fps_clock = pygame.time.Clock()
        self.ball = Ball(20, 20, width/2, height/2)

    def run(self):
        """
        Główna pętla programu
        """
        while not self.handle_events():
            # działaj w pętli do momentu otrzymania sygnału do wyjścia
            self.ball.move()
            self.board.draw(
                self.ball,
            )
            self.fps_clock.tick(30)

    def handle_events(self):
        """
        Obsługa zdarzeń systemowych, tutaj zinterpretujemy np. ruchy myszką

        :return True jeżeli pygame przekazał zdarzenie wyjścia z gry
        """
        for event in pygame.event.get():
            if event.type == pygame.locals.QUIT:
                pygame.quit()
                return True

class Drawable(object):
    """
    Klasa bazowa dla rysowanych obiektów
    """

    def __init__(self, width, height, x, y, color=(0, 255, 0)):
        self.width = width
        self.height = height
        self.color = color
        self.surface = pygame.Surface([width, height], pygame.SRCALPHA, 32).convert_alpha()
        self.rect = self.surface.get_rect(x=x, y=y)

    def draw_on(self, surface):
        surface.blit(self.surface, self.rect)

class Ball(Drawable):
    """
    Piłeczka, sama kontroluje swoją prędkość i kierunek poruszania się.
    """
    def __init__(self, width, height, x, y, color=(255, 0, 0), x_speed=3, y_speed=3):
        super(Ball, self).__init__(width, height, x, y, color)
        pygame.draw.ellipse(self.surface, self.color, [0, 0, self.width, self.height])
        self.x_speed = x_speed
        self.y_speed = y_speed
        self.start_x = x
        self.start_y = y

    def bounce_y(self):
        """
        Odwraca wektor prędkości w osi Y
        """
        self.y_speed *= -1

    def bounce_x(self):
        """
        Odwraca wektor prędkości w osi X
        """
        self.x_speed *= -1

    def reset(self):
        """
        Ustawia piłeczkę w położeniu początkowym i odwraca wektor prędkości w osi Y
        """
        self.rect.move(self.start_x, self.start_y)
        self.bounce_y()

    def move(self):
        """
        Przesuwa piłeczkę o wektor prędkości
        """
        self.rect.x += self.x_speed
        self.rect.y += self.y_speed

# Ta część powinna być zawsze na końcu modułu (ten plik jest modułem)
# chcemy uruchomić naszą grę dopiero po tym jak wszystkie klasy zostaną zadeklarowane
if __name__ == "__main__":
    game = PongGame(800, 400)
    game.run()
<commit_msg>pong z4<commit_after># coding=utf-8

import pygame
import pygame.locals

class Board(object):
    """
    Plansza do gry. Odpowiada za rysowanie okna gry.
    """

    def __init__(self, width, height):
        """
        Konstruktor planszy do gry. Przygotowuje okienko gry.

        :param width:
        :param height:
        """
        self.surface = pygame.display.set_mode((width, height), 0, 32)
        pygame.display.set_caption('Simple Pong')

    def draw(self, *args):
        """
        Rysuje okno gry

        :param args: lista obiektów do narysowania
        """
        background = (230, 255, 255)
        self.surface.fill(background)
        for drawable in args:
            drawable.draw_on(self.surface)

        # dopiero w tym miejscu następuje fatyczne rysowanie
        # w oknie gry, wcześniej tylko ustalaliśmy co i jak ma zostać narysowane
        pygame.display.update()

class PongGame(object):
    """
    Łączy wszystkie elementy gry w całość.
    """

    def __init__(self, width, height):
        pygame.init()
        self.board = Board(width, height)
        # zegar którego użyjemy do kontrolowania szybkości rysowania
        # kolejnych klatek gry
        self.fps_clock = pygame.time.Clock()
        self.ball = Ball(width=20, height=20, x=width/2, y=height/2)

    def run(self):
        """
        Główna pętla programu
        """
        while not self.handle_events():
            # działaj w pętli do momentu otrzymania sygnału do wyjścia
            self.ball.move(self.board)
            self.board.draw(
                self.ball,
            )
            self.fps_clock.tick(30)

    def handle_events(self):
        """
        Obsługa zdarzeń systemowych, tutaj zinterpretujemy np. ruchy myszką

        :return True jeżeli pygame przekazał zdarzenie wyjścia z gry
        """
        for event in pygame.event.get():
            if event.type == pygame.locals.QUIT:
                pygame.quit()
                return True

class Drawable(object):
    """
    Klasa bazowa dla rysowanych obiektów
    """

    def __init__(self, width, height, x, y, color=(0, 255, 0)):
        self.width = width
        self.height = height
        self.color = color
        self.surface = pygame.Surface([width, height], pygame.SRCALPHA, 32).convert_alpha()
        self.rect = self.surface.get_rect(x=x, y=y)

    def draw_on(self, surface):
        surface.blit(self.surface, self.rect)

class Ball(Drawable):
    """
    Piłeczka, sama kontroluje swoją prędkość i kierunek poruszania się.
    """
    def __init__(self, width, height, x, y, color=(255, 0, 0), x_speed=3, y_speed=3):
        super(Ball, self).__init__(width, height, x, y, color)
        pygame.draw.ellipse(self.surface, self.color, [0, 0, self.width, self.height])
        self.x_speed = x_speed
        self.y_speed = y_speed
        self.start_x = x
        self.start_y = y

    def bounce_y(self):
        """
        Odwraca wektor prędkości w osi Y
        """
        self.y_speed *= -1

    def bounce_x(self):
        """
        Odwraca wektor prędkości w osi X
        """
        self.x_speed *= -1

    def reset(self):
        """
        Ustawia piłeczkę w położeniu początkowym i odwraca wektor prędkości w osi Y
        """
        self.rect.move(self.start_x, self.start_y)
        self.bounce_y()

    def move(self, board):
        """
        Przesuwa piłeczkę o wektor prędkości
        """
        self.rect.x += self.x_speed
        self.rect.y += self.y_speed

        if self.rect.x < 0 or self.rect.x > board.surface.get_width():
            self.bounce_x()

        if self.rect.y < 0 or self.rect.y > board.surface.get_height():
            self.bounce_y()

# Ta część powinna być zawsze na końcu modułu (ten plik jest modułem)
# chcemy uruchomić naszą grę dopiero po tym jak wszystkie klasy zostaną zadeklarowane
if __name__ == "__main__":
    game = PongGame(800, 400)
    game.run()
<|endoftext|>


<commit_before>import locale

from django.contrib import admin
from django.core import serializers
from django.utils.html import format_html
from django.utils.translation import gettext_lazy as _

from ..db.models import OwnerModel

class PreviewMixin(object):
    image_width = 200
    image_height = 100

    def get_preview(self, obj):
        html = '
'.join(''.format(
            item.strip(), self.image_width, self.image_height) for item in obj.images.split('\n'))
        return format_html(html)
    get_preview.short_description = _('preview')

class CurrencyMixin(object):

    def format_currency(self, amount, min):
        value = amount / 100
        locale.setlocale(locale.LC_ALL, '')
        formatted = locale.currency(value, False, True)
        if value < min:
            formatted = '{} (低于{})'.format(formatted, locale.currency(min, False, True))
        return format_html(formatted)

class SuperAdmin(CurrencyMixin, PreviewMixin, admin.ModelAdmin):

    def save_model(self, request, obj, form, change):
        if not obj.pk:  # 如果obj.pk不存在, 为新创建
            if 'user_id' in [f.name for f in obj._meta.get_fields()]:
                if not obj.user_id:
                    obj.user_id = request.user.id
            if isinstance(obj, OwnerModel):
                obj.username = request.user.username
                obj.user_avatar = request.user.avatar
        super().save_model(request, obj, form, change)

    def log_change(self, request, obj, message):
        new_message = '{}'.format(serializers.serialize('json', [obj]))
        super().log_change(request, obj, new_message)
<commit_msg>修正format<commit_after>import locale

from django.contrib import admin
from django.core import serializers
from django.utils.html import format_html
from django.utils.translation import gettext_lazy as _

from ..db.models import OwnerModel

class PreviewMixin(object):
    image_width = 200
    image_height = 100

    def get_preview(self, obj):
        html = '
'.join(''.format(
            item.strip(), self.image_width, self.image_height) for item in obj.images.split('\n'))
        return format_html(html)
    get_preview.short_description = _('preview')

class CurrencyMixin(object):

    def format_currency(self, amount, min):
        value = amount / 100
        locale.setlocale(locale.LC_ALL, '')
        locale._override_localeconv = {'n_sign_posn': 1}
        formatted = locale.currency(value, grouping=True)
        if value < min:
            formatted = '{} (低于{})'.format(formatted, locale.currency(min, False, True))
        return format_html(formatted)

class SuperAdmin(CurrencyMixin, PreviewMixin, admin.ModelAdmin):

    def save_model(self, request, obj, form, change):
        if not obj.pk:  # 如果obj.pk不存在, 为新创建
            if 'user_id' in [f.name for f in obj._meta.get_fields()]:
                if not obj.user_id:
                    obj.user_id = request.user.id
            if isinstance(obj, OwnerModel):
                obj.username = request.user.username
                obj.user_avatar = request.user.avatar
        super().save_model(request, obj, form, change)

    def log_change(self, request, obj, message):
        new_message = '{}'.format(serializers.serialize('json', [obj]))
        super().log_change(request, obj, new_message)
<|endoftext|>


<commit_before>import uuid

from gov.cca import AbstractFramework
from gov.cca import Port
from gov.cca.ports import BuilderService
from gov.cca.ports import EventType
from framework.info.connectioninfo import ConnectionID
from framework.info.componentinfo import ComponentID
from framework.common.exceptions import InstanceNotFoundException
from framework.manage.services import ServicesHandle

class ProviderEntry():
   def __init__(self, componentID, serviceProvider):
      self.componentID = componentID
      self.serviceProvider = serviceProvider
      return

class ComponentInstance():
   def __init__(self, component, services):
      self.component = component
      self.services = services

      # Maps a string port name to a gov.cca.ConnectionID
      self.usesConnection = {}

      # Maps a string port name to a set of gov.cca.ConnectionID
      # Has to be initialized with every provides port from the component
      # I'm considering that each component has only one connection to another component through the same uses/provides port.
      self.providesConnection = {}

class FrameworkHandle(AbstractFramework, BuilderService):
   def __init__(self):
      # Maps a string corresponding to a component instance name a ComponentInstance object
      # (instanceName) -> (ComponentInstance)
      self.d_instance = {}

      # (string portType) -> (ProviderEntry)
      self.d_serviceProviders = {}

      # (string portType) -> (gov.cca.Port)
      self.d_servicePorts = []

      # Maps instance names to class names
      # (instanceName) -> (className)
      self.d_aliases = {}

   # New Methods
   def lookupPort(self, componentID, portName):
      """
      Simplifies the access to GoPort on components without requiring the main method to register itself with
      a gov.cca.Services object.
      input: gov.cca.ComponentID componentID, string portName
      output: a gov.cca.Port object
      """
      instanceName = componentID.getInstanceName()
      if instanceName not it self.d_instance.keys():
         raise InstanceNotFound(instanceName)
      return self.d_instance[instanceName].services.getProvidesPort(portName)

   def isProvidedService(self, portType):
      """
      input: a string portType
      output: a boolean
      """
      if portType == "gov.cca.ports.ConnectionEventService" or portType == "gov.cca.ports.ServiceRegistry" :
         return True
      if portType in self.d_serviceProviders.keys() or portType in self.d_servicePorts.keys():
         return True
      return False

   def provideRequestedServices(self, componentID, portName, portType)
      """
      Provides access to two ports implemented by the framework itself: BuilderService and ConnectionEventService
      input: a gov.cca.ComponentID object, a string portName, a string portType
      output: void
      """
      if portType == "gov.cca.ports.ConnectionEventService" :
         userSvcs = self.d_instance[componentID.getInstanceName()].services
         uniqueName = getUniqueName("connectionEventer")
         svcs = getServices(uniqueName, portType, 0)
         svcs.addProvidesPort(userSvcs, "ConnectionEventService", portType, 0)
         connect(componentID, portName, svcs.getComponentID(), "ConnectionEventService")
      else if portType == "gov.cca.ports.ServiceRegistry" :
         userSvcs = self.d_instance[componentID.getInstanceName()].services
         uniqueName = getUniqueName("registryService")
         svcs = getServices(uniqueName, portType, 0)
         svcs.addProvidesPort(userSvcs, "RegistryService", portType, 0)
         connect(componentID, portName, svcs.getComponentID(), "RegistryService")
      else if portType in self.d_servicePorts.keys() :
         port = self.d_servicePorts[portType]
         uniqueName = getUniqueName("singletonPort")
         svcs = getServices(uniqueName, portType, 0)
         svcs.addProvidesPort(port, "AvailService", portType, 0)
         connect(componentID, portName, svcs.getComponentID(), "AvailService")
      else if portType in self.d_serviceProviders.keys() :
         pe = d_servicesProviders[portType]
         sp = pe.serviceProvider
         portName = sp.createService(portType)
         connect(componentID, portName, pe.componentID, portName)
      return

   def addServiceProvider(self, portType, componentID, provider):
      """
      input: a string portType, a gov.cca.ComponenteID componentID, a gov.cca.ports.ServiceProvider provider
      output: none
      """
      pass

   def addServicePort(self, portType, port):
      """
      input: a string portType, gov.cca.Ports port
      output: none
      """
      pass

   def removeFromRegistry(self, portType):
      """
      input: a string portType
      output: none
      """
      pass

   def setInstanceRelease(self, componentID, callback):
      """
      input: a gov.cca.ComponentID componentID, a gov.cca.ComponentRelease callback2
      output: none
      """
      pass

   def getUniqueName(self, requestedName):
      """
      input: a string requestedName
      output: a string that is unique in the framework scope
      """
      return requestedName + "::" + str(uuid.uuid4())

   def removeInstance(self, instanceName):
      """
      input: a string instanceName
      output: a integer
      """
      pass

   # Methods from AbstractFramework
   def createTypeMap(self):
      """
      input: none
      output: a TypeMap object
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def createEmptyFramework(self):
      """
      input: none
      output: a AbstractFramework object
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getServices(self, selfInstanceName, selfClassName, selfProperties):
      """
      input: a string selfInstanceName, string selfClassName, TypeMap selfProperties
      output: a Services object
      throws CCAException
      """
      nil = None
      cid = ComponentID(None)
      uniqueName = getUniqueName(selfInstanceName)
      cid.initialize(uniqueName)
      svcs = ServicesHandle()
      svcs.initialize(self, cid, selfProperties)
      self.d_instance[uniqueName].component = nil;
      self.d_instance[uniqueName].svcs = svcs
      self.d_aliases[uniqueName] = selfClassName
      return svcs

   def releaseServices(self, services):
      """
      input: a Services object
      output: a AbstractFramework object
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def shutdownFramework(self):
      """
      input: none
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   # Methods from BuilderService
   def createInstance(self, instanceName, className, properties):
      """
      input: a string instanceName, a string className, a gov.ccaTypeMap properties
      output: a gov.cca.ComponentID object
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getDeserialization(self, s):
      """
      input: a string s
      output: a gov.cca.ComponentID object
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def connect(self, user, usingPortName, provider, providingPortName):
      """
      input: a gov.cca.ComponentID object user, a string usingPortName, a gov.cca.ComponentID object provider, a string providingPortName
      output: a gov.cca.ConnectionID object
      throws CCAException
      """
      connectionID = ConnectionID()
      userName = user.getInstanceName()
      provName = provider.getInstanceName()
      if (userName not it d_instance.keys()) and (provName not it d_instance.keys()) :
         userSvc = d_instance[userName].services
         provSvc = d_instance[provName].services

         provSvc.notifyConnectionEvent(provindingPortName, EventType.ConnectPending)
         userSvc.notifyConnectionEvent(usingPortName, EventType.ConnectPending)

         port = provSvc.getProvidesPort(providingPortName)

         userSvc.bindPort(usingPortName, Port)
         connectID = ConnectionID()
         connectID.initialize(provider, providingPortName, user, usingPortName, 0)

         d_instance[userName].usesConnection[usingPortName] = connectID
         d_instance[provName].providesConnection[provindingPortName].append(connectID)

         provSvc.notifyConnectionEvent(providingPortName, EventType.Connected)
         usesSvc.notifyConnectionEvent(usingPortName, EventType.Connected)
    return connectionID

   def disconnect(self, connID, timeout):
      """
      input: a gov.cca.ConnectionID object connID, a float timeout
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def disconnectAll(self, id1, id2, timeout):
      """
      input: a gov.cca.ComponentID id1, a gov.cca.ComponentID id2, a float timeout
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def destroyInstance(self, toDie, timeout):
      """
      input: a gov.cca.ComponentID toDie, a float timeout
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getComponentProperties(self, cid):
      """
      input: a gov.cca.ComponentID cid
      output: a gov.cca.TypeMap
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def setComponentProperties(self, cid, properties):
      """
      input: a gov.cca.ComponentID cid, a gov.cca.TypeMap properties
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getPortProperties(self, cid, portName):
      """
      input: a gov.cca.ComponentID cid, a string portName
      output: a gov.cca.TypeMap
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def setPortProperties(self, cid, portName, properties):
      """
      input: a gov.cca.ComponentID cid, a string portName, a gov.cca.TypeMap properties
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getConnectionProperties(self, connID):
      """
      input: a gov.cca.ConnectionID connID
      output: a gov.cca.TypeMap
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def setConnectionProperties(self, connID, properties):
      """
      input: a gov.cca.ConnectionID connID, a gov.cca.TypeMap properties
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getComponentID(self, componentInstanceName):
      """
      input: a string componentInstanceName
      output: a gov.cca.ComponentID
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getComponentIDs(self):
      """
      input: none
      output: a list of gov.cca.ComponentID
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getProvidedPortNames(self, cid):
      """
      input: a gov.cca.ComponenID cid
      output: a list of strings
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getUsedPortNames(self, cid):
      """
      input: a gov.cca.ComponenID cid
      output: a list of strings
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getConnectionIDs(self, componentList):
      """
      input: a list of gov.cca.ComponentID
      output: a list of goc.cca.ConnectionID
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

<commit_msg>Halfway through the framework zone!!!<commit_after>import uuid
import importlib

from gov.cca import AbstractFramework
from gov.cca import Port
from gov.cca.ports import BuilderService
from gov.cca.ports import EventType
from framework.info.connectioninfo import ConnectionID
from framework.info.componentinfo import ComponentID
from framework.common.exceptions import InstanceNotFoundException
from framework.manage.services import ServicesHandle
from framework.common.typemap import TypeMapDict

class ProviderEntry():
   def __init__(self, componentID, serviceProvider):
      self.componentID = componentID
      self.serviceProvider = serviceProvider
      return

class ComponentInstance():
   def __init__(self, component, release, services):
      self.component = component
      self.release = release
      self.services = services

      # Maps a string port name to a gov.cca.ConnectionID
      self.usesConnection = {}

      # Maps a string port name to a set of gov.cca.ConnectionID
      # Has to be initialized with every provides port from the component
      # I'm considering that each component has only one connection to another component through the same uses/provides port.
      self.providesConnection = {}

class FrameworkHandle(AbstractFramework, BuilderService):
   def __init__(self):
      # Maps a string corresponding to a component instance name a ComponentInstance object
      # (instanceName) -> (ComponentInstance)
      self.d_instance = {}

      # (string portType) -> (ProviderEntry)
      self.d_serviceProviders = {}

      # (string portType) -> (gov.cca.Port)
      self.d_servicePorts = []

      # Maps instance names to class names
      # (instanceName) -> (className)
      self.d_aliases = {}

   # New Methods
   def lookupPort(self, componentID, portName):
      """
      Simplifies the access to GoPort on components without requiring the main method to register itself with
      a gov.cca.Services object.
      input: gov.cca.ComponentID componentID, string portName
      output: a gov.cca.Port object
      """
      instanceName = componentID.getInstanceName()
      if instanceName not it self.d_instance.keys():
         raise InstanceNotFound(instanceName)
      return self.d_instance[instanceName].services.getProvidesPort(portName)

   def isProvidedService(self, portType):
      """
      input: a string portType
      output: a boolean
      """
      if portType == "gov.cca.ports.ConnectionEventService" or portType == "gov.cca.ports.ServiceRegistry" :
         return True
      if portType in self.d_serviceProviders.keys() or portType in self.d_servicePorts.keys():
         return True
      return False

   def provideRequestedServices(self, componentID, portName, portType)
      """
      Provides access to two ports implemented by the framework itself: BuilderService and ConnectionEventService
      input: a gov.cca.ComponentID object, a string portName, a string portType
      output: void
      """
      if portType == "gov.cca.ports.ConnectionEventService" :
         userSvcs = self.d_instance[componentID.getInstanceName()].services
         uniqueName = getUniqueName("connectionEventer")
         svcs = getServices(uniqueName, portType, 0)
         svcs.addProvidesPort(userSvcs, "ConnectionEventService", portType, 0)
         connect(componentID, portName, svcs.getComponentID(), "ConnectionEventService")
      else if portType == "gov.cca.ports.ServiceRegistry" :
         userSvcs = self.d_instance[componentID.getInstanceName()].services
         uniqueName = getUniqueName("registryService")
         svcs = getServices(uniqueName, portType, 0)
         svcs.addProvidesPort(userSvcs, "RegistryService", portType, 0)
         connect(componentID, portName, svcs.getComponentID(), "RegistryService")
      else if portType in self.d_servicePorts.keys() :
         port = self.d_servicePorts[portType]
         uniqueName = getUniqueName("singletonPort")
         svcs = getServices(uniqueName, portType, 0)
         svcs.addProvidesPort(port, "AvailService", portType, 0)
         connect(componentID, portName, svcs.getComponentID(), "AvailService")
      else if portType in self.d_serviceProviders.keys() :
         pe = d_servicesProviders[portType]
         sp = pe.serviceProvider
         portName = sp.createService(portType)
         connect(componentID, portName, pe.componentID, portName)
      return

   def addServiceProvider(self, portType, componentID, provider):
      """
      input: a string portType, a gov.cca.ComponenteID componentID, a gov.cca.ports.ServiceProvider provider
      output: none
      """
      pe = ProviderEntry(componentID, provider)
      self.d_serviceProviders[portType] = pe
      return

   def addServicePort(self, portType, port):
      """
      input: a string portType, gov.cca.Ports port
      output: none
      """
      self.d_servicePorts[portType] = port
      return

   def removeFromRegistry(self, portType):
      """
      input: a string portType
      output: none
      """
      if portType != "gov.cca.ports.BuilderService":
         self.d_servicePorts.pop(portType, None)
         self.d_serviceProviders.pop(portType, None
      return

   def setInstanceRelease(self, componentID, callback):
      """
      input: a gov.cca.ComponentID componentID, a gov.cca.ComponentRelease callback2
      output: none
      """
      instanceName = componentID.getInstanceName()
      self.d_instance[instanceName].release = callback
      return

   def getUniqueName(self, requestedName):
      """
      input: a string requestedName
      output: a string that is unique in the framework scope
      """
      return requestedName + "::" + str(uuid.uuid4())

   def removeInstance(self, instanceName):
      """
      input: a string instanceName
      output: a integer
      """
      if instanceName not in self.d_instance.keys():
         return 0

      connectionIDs = []

      # Collect all connection IDs
      for connection in self.d_instance[instanceName].usesConnection:
         connectionIDs.append(self.d_instance[instanceName].usesConnection[connection])

      # Destroy all connections
      for id_ in connectionIDs:
         disconnect(id_, 0.0)

      # Remove the instance itself
      instance = self.d_instance[instanceName]
      instance.release.releaseServices(instance.services)
      self.d_instance.pop(instanceName, None)

      return 1

   # Methods from AbstractFramework
   def createTypeMap(self):
      """
      input: none
      output: a TypeMap object
      throws CCAException
      """
      return TypeMapDict()

   def createEmptyFramework(self):
      """
      input: none
      output: a AbstractFramework object
      throws CCAException
      """
      return FrameworkHandle()

   def getServices(self, selfInstanceName, selfClassName, selfProperties):
      """
      input: a string selfInstanceName, string selfClassName, TypeMap selfProperties
      output: a Services object
      throws CCAException
      """
      nil = None
      cid = ComponentID(None)
      uniqueName = getUniqueName(selfInstanceName)
      cid.initialize(uniqueName)
      svcs = ServicesHandle()
      svcs.initialize(self, cid, selfProperties, True)
      self.d_instance[uniqueName].component = None
      self.d_instance[uniqueName].services = svcs
      self.d_aliases[uniqueName] = selfClassName
      return svcs

   def releaseServices(self, services):
      """
      input: a Services object
      output: a AbstractFramework object
      throws CCAException
      """
      if services != None:
         cid = services.getComponentID()
         instanceName = cid.getInstanceName()
         if instanceName in self.d_aliases.keys() :
            n_removed_instances = removeInstance(instanceName)
            n_removed_aliases = self.d_aliases.pop(instanceName, None)
            if n_removed_instances != 1 or n_remove_aliases == None :
               print "Unexpected behavior removind instances."
               print "n_removed_instances: " + n_removed_instances
               print "n_removed_aliases: " + n_removed_aliases
         else :
            print "Error: releaseServices() called on services object not created by getServices()"
      return

   def shutdownFramework(self):
      """
      input: none
      output: void
      throws CCAException
      """
      for instanceName in self.d_instance:
         removeInstance(instanceName)

   # Methods from BuilderService
   def createInstance(self, instanceName, className, properties):
      """
      Our components are Python classes. The className is in the format xxx.xxx.xxx.Class, where the xxx.
      stands for the module to be imported and the Class is the name of the class. The xxx. part may be
      repeated. For example, in doe.cca.Library.GaussianElimination, doe.cca.Library is the module name
      and GaussianElimination is the class.
      We could, in theory, use the properties TypeMap to pass parameters to the component object constructor.
      But I will not do it. Let the calling entity call a initialize method.
      input: a string instanceName, a string className, a gov.cca.TypeMap properties
      output: a gov.cca.ComponentID object
      throws CCAException
      """
      moduleName = className.split('.')[0:-1]
      moduleName = ".".join(moduleName)
      className_ = className.split('.')[-1]
      class_ =  getattr(importlib.import_module(moduleName), className_)
      component = class_()

      uniqueName = getUniqueName(instanceName)
      cid = ComponentID(None)
      cid.initiliaze(uniqueName)
      services = ServicesHandle()
      services.initialize(self, cid, properties, False)
      self.d_instance[uniqueName].component = component
      self.d_instance[uniqueName].services = services
      component.setServices(services)

      return cid

   def getDeserialization(self, s):
      """
      input: a string s
      output: a gov.cca.ComponentID object
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def connect(self, user, usingPortName, provider, providingPortName):
      """
      input: a gov.cca.ComponentID object user, a string usingPortName, a gov.cca.ComponentID object provider, a string providingPortName
      output: a gov.cca.ConnectionID object
      throws CCAException
      """
      connectionID = ConnectionID()
      userName = user.getInstanceName()
      provName = provider.getInstanceName()
      if (userName not it d_instance.keys()) and (provName not it d_instance.keys()) :
         userSvc = d_instance[userName].services
         provSvc = d_instance[provName].services

         provSvc.notifyConnectionEvent(provindingPortName, EventType.ConnectPending)
         userSvc.notifyConnectionEvent(usingPortName, EventType.ConnectPending)

         port = provSvc.getProvidesPort(providingPortName)

         userSvc.bindPort(usingPortName, Port)
         connectID = ConnectionID()
         connectID.initialize(provider, providingPortName, user, usingPortName, 0)

         d_instance[userName].usesConnection[usingPortName] = connectID
         d_instance[provName].providesConnection[provindingPortName].append(connectID)

         provSvc.notifyConnectionEvent(providingPortName, EventType.Connected)
         usesSvc.notifyConnectionEvent(usingPortName, EventType.Connected)
    return connectionID

   def disconnect(self, connID, timeout):
      """
      input: a gov.cca.ConnectionID object connID, a float timeout
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def disconnectAll(self, id1, id2, timeout):
      """
      input: a gov.cca.ComponentID id1, a gov.cca.ComponentID id2, a float timeout
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def destroyInstance(self, toDie, timeout):
      """
      input: a gov.cca.ComponentID toDie, a float timeout
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getComponentProperties(self, cid):
      """
      input: a gov.cca.ComponentID cid
      output: a gov.cca.TypeMap
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def setComponentProperties(self, cid, properties):
      """
      input: a gov.cca.ComponentID cid, a gov.cca.TypeMap properties
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getPortProperties(self, cid, portName):
      """
      input: a gov.cca.ComponentID cid, a string portName
      output: a gov.cca.TypeMap
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def setPortProperties(self, cid, portName, properties):
      """
      input: a gov.cca.ComponentID cid, a string portName, a gov.cca.TypeMap properties
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getConnectionProperties(self, connID):
      """
      input: a gov.cca.ConnectionID connID
      output: a gov.cca.TypeMap
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def setConnectionProperties(self, connID, properties):
      """
      input: a gov.cca.ConnectionID connID, a gov.cca.TypeMap properties
      output: void
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getComponentID(self, componentInstanceName):
      """
      input: a string componentInstanceName
      output: a gov.cca.ComponentID
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getComponentIDs(self):
      """
      input: none
      output: a list of gov.cca.ComponentID
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getProvidedPortNames(self, cid):
      """
      input: a gov.cca.ComponenID cid
      output: a list of strings
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getUsedPortNames(self, cid):
      """
      input: a gov.cca.ComponenID cid
      output: a list of strings
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

   def getConnectionIDs(self, componentList):
      """
      input: a list of gov.cca.ComponentID
      output: a list of goc.cca.ConnectionID
      throws CCAException
      """
      raise NotImplementedError("Abstract Class!")

<|endoftext|>


<commit_before>from django.test import TestCase
from api.models import *

# Create your tests here.
class Models_Tests(TestCase):
    def test_volunteer(self):
        store = Volunteer(name="Ian", email="ian@ianluo.com",
                          phone="1231231234", city="medford",
                          jacket="yes", jacket_size="M")
        self.assertEqual(store.name, "Ian")
        store.save() # store into the db

        retrieved = Volunteer.objects.get(name="Ian")
        self.assertEqual(store.email, retrieved.email)

    def test_event(self):
        store = Event(name="Boston Marathon")
        store.date = "12/02/1994" # this is my birthday!
        store.save()

        retrieve = Event.objects.get(name="Boston Marathon")
        self.assertEqual(retrieve.date, "12/02/1994")

    def test_attendee(self):
        self.test_volunteer()
        self.test_event()

        ian = Volunteer.objects.get(name="Ian")
        ians_bday = Event.objects.get(name="Boston Marathon")
        john_cena = Volunteer(name="John Cena").save()

        new_attendee = Attendee(volunteer=ian, event=ians_bday, at_event=True, notes="Was lit", team_captain=john_cena)
        new_attendee.save()

        retrieve = Attendee.objects.get(notes="Was lit")
        self.assertEqual(retrieve.volunteer.name, "Ian")<commit_msg>modify test<commit_after>from django.test import TestCase
from api.models import *

# Create your tests here.
class Models_Tests(TestCase):
    def test_volunteer(self):
        store = Volunteer(name="Ian", email="ian@ianluo.com",
                          phone="1231231234", city="medford",
                          jacket="yes", jacket_size="M")
        self.assertEqual(store.name, "Ian")
        store.save() # store into the db

        retrieved = Volunteer.objects.get(name="Ian")
        self.assertEqual(store.email, retrieved.email)

    def test_event(self):
        store = Event(name="Boston Marathon")
        store.date = "12/02/1994" # this is my birthday!
        store.save()

        retrieve = Event.objects.get(name="Boston Marathon")
        self.assertEqual(retrieve.date, "12/02/1994")

    def test_attendee(self):
        self.test_volunteer()
        self.test_event()

        ian = Volunteer.objects.get(name="Ian")
        ians_bday = Event.objects.get(name="Boston Marathon")
        john_cena = Volunteer(name="John Cena").save()

        new_attendee = Attendee(volunteer=ian, event=ians_bday, at_event=True, notes="Was lit", team_captain=john_cena)
        new_attendee.save()

        retrieve = Attendee.objects.get(notes="Was lit")
        self.assertEqual(retrieve.volunteer.name, "Ian")

class Integration_tests(TestCase):
    def check_homepage(self):
        pass
        <|endoftext|>


<commit_before>"""Available platforms"""

__author__ = "Felix Simkovic"
__date__ = "03 Jun 2017"
__version__ = "0.1"

import sys

# OS-dependent script headers and extensions
if sys.platform.startswith('win'):
    EXE_EXT, SCRIPT_HEADER, SCRIPT_EXT = ('.exe', '', '.bat')
else:
    EXE_EXT, SCRIPT_HEADER, SCRIPT_EXT = ('', '#!/bin/bash', '.sh')

# Little factory function to get the right platform without importing all
def platform_factory(qtype):
    """Return the correct platform handler"""
    qtype = qtype.lower()
    if qtype == "local":
        from pyjob.platform.local import LocalJobServer
        return = LocalJobServer
    elif qtype == "lsf":
        from pyjob.platform.lsf import LoadSharingFacility
        return LoadSharingFacility
    elif qtype == "sge":
        from pyjob.platform.sge import SunGridEngine
        return SunGridEngine
    else:
        raise ValueError("Unknown platform")

<commit_msg>typo corrected<commit_after>"""Available platforms"""

__author__ = "Felix Simkovic"
__date__ = "03 Jun 2017"
__version__ = "0.1"

import sys

# OS-dependent script headers and extensions
if sys.platform.startswith('win'):
    EXE_EXT, SCRIPT_HEADER, SCRIPT_EXT = ('.exe', '', '.bat')
else:
    EXE_EXT, SCRIPT_HEADER, SCRIPT_EXT = ('', '#!/bin/bash', '.sh')

# Little factory function to get the right platform without importing all
def platform_factory(qtype):
    """Return the correct platform handler"""
    qtype = qtype.lower()
    if qtype == "local":
        from pyjob.platform.local import LocalJobServer
        return LocalJobServer
    elif qtype == "lsf":
        from pyjob.platform.lsf import LoadSharingFacility
        return LoadSharingFacility
    elif qtype == "sge":
        from pyjob.platform.sge import SunGridEngine
        return SunGridEngine
    else:
        raise ValueError("Unknown platform")

<|endoftext|>


<commit_before>"""
Dataset preloading tool

This file provides the ability to make a local cache of a dataset or part of
it. It is meant to help in the case where multiple jobs are reading the same
dataset from ${PYLEARN2_DATA_PATH}, which may cause a great burden on the
network.

With this file, it is possible to make a local copy
(in ${PYLEARN2_LOCAL_DATA_PATH}) of any required file and have multiple
processes use it simultaneously instead of each acquiring its own copy
over the network.

Whenever a folder or a dataset copy is created locally, it is granted
the same access as it has under ${PYLEARN2_LOCAL_DATA_PATH}. This is
gauranteed by default copy.

"""

import os
import time
import atexit
import logging
from pylearn2.utils import string_utils
import theano.gof.compilelock as compilelock

log = logging.getLogger(__name__)

class LocalDatasetCache:

    def __init__(self):
        default_path = '${PYLEARN2_DATA_PATH}'
        local_path = '${PYLEARN2_LOCAL_DATA_PATH}'
        self.pid = os.getpid()

        try:
            self.dataset_remote_dir = string_utils.preprocess(default_path)
            self.dataset_local_dir = string_utils.preprocess(local_path)
        except:
            # Local cache seems to be deactivated
            self.dataset_remote_dir = ""
            self.dataset_local_dir = ""

    def cache_file(self, filename):
        """
        Caches a file locally if possible. If caching was succesfull, or if
        the file was previously successfully cached, this method returns the
        path to the local copy of the file. If not, it returns the path to
        the original file.
        """

        remote_name = string_utils.preprocess(filename)

        # Check if a local directory for data has been defined. Otherwise,
        # do not locally copy the data
        if self.dataset_local_dir == "":
            log.warning("Local cache deactivated : file %s not cached" %
                        remote_name)
            return filename

        # Make sure the file to cache exists and really is a file
        if not os.path.exists(remote_name):
            log.error("Error : Specified file %s does not exist" %
                      remote_name)
            return filename

        if not os.path.isfile(remote_name):
            log.error("Error : Specified name %s is not a file" %
                      remote_name)
            return filename

        # Create the $PYLEARN2_LOCAL_DATA_PATH folder if needed
        self.safe_mkdir(self.dataset_local_dir)

        # Determine local path to which the file is to be cached
        local_name = os.path.join(self.dataset_local_dir,
                                  os.path.relpath(remote_name,
                                                  self.dataset_remote_dir))

        # Create the folder structure to receive the remote file
        local_folder = os.path.split(local_name)[0]
        self.safe_mkdir(local_folder)

        # Acquire writelock on the local file to prevent the possibility
        # of any other process modifying it while we cache it if needed.
        # Also, if another process is currently caching the same file,
        # it forces the current process to wait for it to be done before
        # using the file.
        self.get_writelock(local_name)

        # If the file does not exist locally, consider creating it
        if not os.path.exists(local_name):

            # Check that there is enough space to cache the file
            if not self.check_enough_space(remote_name, local_name):
                log.warning("Not enough free space : file %s not cached" %
                            remote_name)
                self.release_writelock()
                return filename

            # There is enough space; make a local copy of the file
            self.copy_from_server_to_local(remote_name, local_name)
            log.info("File %s has been locally cached to %s" %
                     (remote_name, local_name))

        else:
            log.info("File %s has previously been locally cached to %s" %
                     (remote_name, local_name))

        # Obtain a readlock on the downloaded file before releasing the
        # writelock. This is to prevent having a moment where there is no
        # lock on this file which could give the impression that it is
        # unused and therefore safe to delete.
        self.get_readlock(local_name)
        self.release_writelock()

        return local_name

    def copy_from_server_to_local(self, remote_fname, local_fname):
        """
        Copies a remote file locally
        """

        head, tail = os.path.split(local_fname)
        head += os.path.sep
        if not os.path.exists(head):
            os.makedirs(os.path.dirname(head))

        command = 'cp ' + remote_fname + ' ' + local_fname
        os.system(command)

    def disk_usage(self, path):
        """
        Return free usage about the given path, in bytes
        """

        st = os.statvfs(path)
        total = st.f_blocks * st.f_frsize
        used = (st.f_blocks - st.f_bfree) * st.f_frsize
        return total, used

    def check_enough_space(self, remote_fname, local_fname,
                           max_disk_usage=0.9):
        """
        Check if the local disk has enough space to store the dataset
        """

        storage_need = os.path.getsize(remote_fname)
        storage_total, storage_used = self.disk_usage(self.dataset_local_dir)

        # Instead of only looking if there's enough space, we ensure we do not
        # go over max disk usage level to avoid filling the disk/partition
        return ((storage_used + storage_need) <
                (storage_total * max_disk_usage))

    def safe_mkdir(self, folderName):
        """
        Create the specified folder. If the parent folders do not
        exist, they are also created. If the folder already exists,
        nothing is done.
        """
        if not os.path.exists(folderName):
            os.makedirs(folderName)

    def get_readlock(self, path):
        """
        Obtain a readlock on a file
        """

        timestamp = int(time.time() * 1e6)
        lockdirName = "%s.readlock.%i.%i" % (path, self.pid, timestamp)
        os.mkdir(lockdirName)

        # Register function to release the readlock at the end of the script
        atexit.register(self.release_readlock, lockdirName=lockdirName)

    def release_readlock(self, lockdirName):
        """
        Release a previously obtained readlock
        """

        # Make sure the lock still exists before deleting it
        if (os.path.exists(lockdirName) and os.path.isdir(lockdirName)):
            os.rmdir(lockdirName)

    def get_writelock(self, filename):
        """
        Obtain a writelock on a file.
        Only one write lock may be held at any given time.
        """

        # compilelock expect locks to be on folder. Since we want a lock on a
        # file, we will have to ask compilelock for a folder with a different
        # name from the file we want a lock on or else compilelock will
        # try to create a folder with the same name as the file
        compilelock.get_lock(filename + ".writelock")

    def release_writelock(self):
        """
        Release the previously obtained writelock
        """
        compilelock.release_lock()

datasetCache = LocalDatasetCache()<commit_msg>Created class docstring<commit_after>"""
Dataset preloading tool

This file provides the ability to make a local cache of a dataset or
part of it. It is meant to help in the case where multiple jobs are
reading the same dataset from ${PYLEARN2_DATA_PATH}, which may cause a
great burden on the network.

With this file, it is possible to make a local copy
(in ${PYLEARN2_LOCAL_DATA_PATH}) of any required file and have multiple
processes use it simultaneously instead of each acquiring its own copy
over the network.

Whenever a folder or a dataset copy is created locally, it is granted
the same access as it has under ${PYLEARN2_LOCAL_DATA_PATH}. This is
gauranteed by default copy.

"""

import os
import time
import atexit
import logging
from pylearn2.utils import string_utils
import theano.gof.compilelock as compilelock

log = logging.getLogger(__name__)

class LocalDatasetCache:
    """
    A local cache for remote files for faster access and reducing
    network stress.
    """

    def __init__(self):
        default_path = '${PYLEARN2_DATA_PATH}'
        local_path = '${PYLEARN2_LOCAL_DATA_PATH}'
        self.pid = os.getpid()

        try:
            self.dataset_remote_dir = string_utils.preprocess(default_path)
            self.dataset_local_dir = string_utils.preprocess(local_path)
        except:
            # Local cache seems to be deactivated
            self.dataset_remote_dir = ""
            self.dataset_local_dir = ""

    def cache_file(self, filename):
        """
        Caches a file locally if possible. If caching was succesfull, or if
        the file was previously successfully cached, this method returns the
        path to the local copy of the file. If not, it returns the path to
        the original file.
        """

        remote_name = string_utils.preprocess(filename)

        # Check if a local directory for data has been defined. Otherwise,
        # do not locally copy the data
        if self.dataset_local_dir == "":
            log.warning("Local cache deactivated : file %s not cached" %
                        remote_name)
            return filename

        # Make sure the file to cache exists and really is a file
        if not os.path.exists(remote_name):
            log.error("Error : Specified file %s does not exist" %
                      remote_name)
            return filename

        if not os.path.isfile(remote_name):
            log.error("Error : Specified name %s is not a file" %
                      remote_name)
            return filename

        # Create the $PYLEARN2_LOCAL_DATA_PATH folder if needed
        self.safe_mkdir(self.dataset_local_dir)

        # Determine local path to which the file is to be cached
        local_name = os.path.join(self.dataset_local_dir,
                                  os.path.relpath(remote_name,
                                                  self.dataset_remote_dir))

        # Create the folder structure to receive the remote file
        local_folder = os.path.split(local_name)[0]
        self.safe_mkdir(local_folder)

        # Acquire writelock on the local file to prevent the possibility
        # of any other process modifying it while we cache it if needed.
        # Also, if another process is currently caching the same file,
        # it forces the current process to wait for it to be done before
        # using the file.
        self.get_writelock(local_name)

        # If the file does not exist locally, consider creating it
        if not os.path.exists(local_name):

            # Check that there is enough space to cache the file
            if not self.check_enough_space(remote_name, local_name):
                log.warning("Not enough free space : file %s not cached" %
                            remote_name)
                self.release_writelock()
                return filename

            # There is enough space; make a local copy of the file
            self.copy_from_server_to_local(remote_name, local_name)
            log.info("File %s has been locally cached to %s" %
                     (remote_name, local_name))

        else:
            log.info("File %s has previously been locally cached to %s" %
                     (remote_name, local_name))

        # Obtain a readlock on the downloaded file before releasing the
        # writelock. This is to prevent having a moment where there is no
        # lock on this file which could give the impression that it is
        # unused and therefore safe to delete.
        self.get_readlock(local_name)
        self.release_writelock()

        return local_name

    def copy_from_server_to_local(self, remote_fname, local_fname):
        """
        Copies a remote file locally
        """

        head, tail = os.path.split(local_fname)
        head += os.path.sep
        if not os.path.exists(head):
            os.makedirs(os.path.dirname(head))

        command = 'cp ' + remote_fname + ' ' + local_fname
        os.system(command)

    def disk_usage(self, path):
        """
        Return free usage about the given path, in bytes
        """

        st = os.statvfs(path)
        total = st.f_blocks * st.f_frsize
        used = (st.f_blocks - st.f_bfree) * st.f_frsize
        return total, used

    def check_enough_space(self, remote_fname, local_fname,
                           max_disk_usage=0.9):
        """
        Check if the local disk has enough space to store the dataset
        """

        storage_need = os.path.getsize(remote_fname)
        storage_total, storage_used = self.disk_usage(self.dataset_local_dir)

        # Instead of only looking if there's enough space, we ensure we do not
        # go over max disk usage level to avoid filling the disk/partition
        return ((storage_used + storage_need) <
                (storage_total * max_disk_usage))

    def safe_mkdir(self, folderName):
        """
        Create the specified folder. If the parent folders do not
        exist, they are also created. If the folder already exists,
        nothing is done.
        """
        if not os.path.exists(folderName):
            os.makedirs(folderName)

    def get_readlock(self, path):
        """
        Obtain a readlock on a file
        """

        timestamp = int(time.time() * 1e6)
        lockdirName = "%s.readlock.%i.%i" % (path, self.pid, timestamp)
        os.mkdir(lockdirName)

        # Register function to release the readlock at the end of the script
        atexit.register(self.release_readlock, lockdirName=lockdirName)

    def release_readlock(self, lockdirName):
        """
        Release a previously obtained readlock
        """

        # Make sure the lock still exists before deleting it
        if (os.path.exists(lockdirName) and os.path.isdir(lockdirName)):
            os.rmdir(lockdirName)

    def get_writelock(self, filename):
        """
        Obtain a writelock on a file.
        Only one write lock may be held at any given time.
        """

        # compilelock expect locks to be on folder. Since we want a lock on a
        # file, we will have to ask compilelock for a folder with a different
        # name from the file we want a lock on or else compilelock will
        # try to create a folder with the same name as the file
        compilelock.get_lock(filename + ".writelock")

    def release_writelock(self):
        """
        Release the previously obtained writelock
        """
        compilelock.release_lock()

datasetCache = LocalDatasetCache()<|endoftext|>


<commit_before>import logging
import os

from pylons import request, response, session, tmpl_context as c
from pylons.controllers.util import abort, redirect_to
from pylons import app_globals as ag

from rdfdatabank.lib.base import BaseController, render
from rdfdatabank.lib.utils import create_new
from rdfdatabank.lib.file_unpack import check_file_mimetype, BadZipfile, get_zipfiles_in_dataset, unpack_zip_item, read_zipfile
from rdfdatabank.lib.conneg import MimeType as MT, parse as conneg_parse

log = logging.getLogger(__name__)

class ItemsController(BaseController):
    def siloview(self, silo):
        abort(403, "Forbidden")

    def datasetview(self, silo, id):
        #tmpl_context variables needed: c.silo_name, c.zipfiles, c.ident, c.id, c.path

        c.silo_name = silo
        c.id = id

        if not request.environ.get('repoze.who.identity'):
            abort(401, "Not Authorised")

        ident = request.environ.get('repoze.who.identity')
        c.ident = ident
        granary_list = ag.granary.silos
        if ident:
            silos = ag.authz(granary_list, ident)
            if silo not in silos:
                abort(403, "Forbidden")
        else:
            abort(403, "Forbidden")

        rdfsilo = ag.granary.get_rdf_silo(silo)
        if not rdfsilo.exists(id):
            abort (403, "Forbidden")

        dataset = rdfsilo.get_item(id)

        http_method = request.environ['REQUEST_METHOD']
        if http_method == "GET":
            c.zipfiles = get_zipfiles_in_dataset(dataset)
            # conneg return
            accept_list = None
            if 'HTTP_ACCEPT' in request.environ:
                accept_list = conneg_parse(request.environ['HTTP_ACCEPT'])
            if not accept_list:
                accept_list= [MT("text", "html")]
            mimetype = accept_list.pop(0)
            while(mimetype):
                if str(mimetype).lower() in ["text/html", "text/xhtml"]:
                    return render("/list_of_zipfiles.html")
                elif str(mimetype).lower() in ["text/plain", "application/json"]:
                    response.content_type = 'application/json; charset="UTF-8"'
                    response.status_int = 200
                    response.status = "200 OK"
                    return simplejson.dumps(dict(c.zipfiles))
                try:
                    mimetype = accept_list.pop(0)
                except IndexError:
                    mimetype = None
            #Whoops nothing satisfies - return text/html
            return render("/list_of_zipfiles.html")
        elif http_method == "POST":
            params = request.POST
            if not (params.has_key("filename") and params['filename']):
                abort(400, "You must supply a filename to unpack")
            item_real_filepath = dataset.to_dirpath()
            target_filepath = "%s/%s"%(item_real_filepath, params['filename'])
            if not os.path.isfile(target_filepath):
                abort(404, "File to unpack not found")
            if not check_file_mimetype(target_filepath, 'application/zip'):
                abort(415, "File is not of type application/zip")

            if params.has_key("id") and params['id']:
                target_dataset_name = params['id']
            else:
                (head, fn) = os.path.split(params['filename'])
                (fn, ext) = os.path.splitext(fn)
                target_dataset_name = "%s-%s"%(id,fn)

            if not rdfsilo.exists(target_dataset_name):
                target_dataset = create_new(rdfsilo, target_dataset_name, ident['repoze.who.userid'])
            else:
                target_dataset = rdfsilo.get_item(target_dataset_name)

            try:
                unpack_zip_item(target_dataset, dataset, params['filename'], rdfsilo, ident['repoze.who.userid'])
            except BadZipfile:
                abort(400, "Couldn't unpack zipfile")

            # conneg return
            accept_list = None
            if 'HTTP_ACCEPT' in request.environ:
                accept_list = conneg_parse(request.environ['HTTP_ACCEPT'])
            if not accept_list:
                accept_list= [MT("text", "html")]
            mimetype = accept_list.pop(0)
            while(mimetype):
                if str(mimetype).lower() in ["text/html", "text/xhtml"]:
                    # probably a browser - redirect to newly created dataset
                    redirect_to(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name)
                elif str(mimetype).lower() in ["text/plain", "application/json"]:
                    response.content_type = "text/plain"
                    response.status_int = 201
                    response.status = "201 Created"
                    #new_item = rdfsilo.get_item(target_dataset_name)
                    #response.headers["Content-Location"] = new_item.uri
                    #response.headers.add("Content-Location", new_item.uri)
                    #response.content_location = item.uri
                    #response.headers['location'] = item.uri
                    #response.location = item.uri
                    return "Created"
                try:
                    mimetype = accept_list.pop(0)
                except IndexError:
                    mimetype = None
            # Whoops - nothing satisfies - return text/plain
            response.content_type = "text/plain"
            response.status_int = 201
            response.status = "201 Created"
            #new_item = rdfsilo.get_item(target_dataset_name)
            #response.headers.add("Content-Location", new_item.uri)
            return "Created"

    def itemview(self, silo, id, path):
        #tmpl_context variables needed: c.silo_name, c.zipfile_contents c.ident, c.id, c.path
        c.silo_name = silo
        c.id = id
        c.path = path

        if not request.environ.get('repoze.who.identity'):
            abort(401, "Not Authorised")

        if not path:
            abort(400, "You must supply a filename to unpack")

        ident = request.environ.get('repoze.who.identity')
        c.ident = ident
        granary_list = ag.granary.silos
        if ident:
            silos = ag.authz(granary_list, ident)
            if silo not in silos:
                abort(403, "Forbidden")
        else:
            abort(403, "Forbidden")

        rdfsilo = ag.granary.get_rdf_silo(silo)
        if not rdfsilo.exists(id):
            abort (403, "Forbidden")

        dataset = rdfsilo.get_item(id)
        item_real_filepath = dataset.to_dirpath()
        target_filepath = "%s/%s"%(item_real_filepath, path)
        #c.parts = dataset.list_parts(detailed=False)
        if not dataset.isfile(path):
            abort(404, "File not found")
        if not os.path.isfile(target_filepath):
            abort(404, "File not found")
        if not check_file_mimetype(target_filepath, 'application/zip'):
            abort(415, "File is not of type application/zip")

        http_method = request.environ['REQUEST_METHOD']
        if http_method == "GET":
            try:
                c.zipfile_contents = read_zipfile(target_filepath)
            except BadZipfile:
                abort(400, "Could not read zipfile")
            # conneg return
            accept_list = None
            if 'HTTP_ACCEPT' in request.environ:
                accept_list = conneg_parse(request.environ['HTTP_ACCEPT'])
            if not accept_list:
                accept_list= [MT("text", "html")]
            mimetype = accept_list.pop(0)
            while(mimetype):
                if str(mimetype).lower() in ["text/html", "text/xhtml"]:
                    return render("/zipfileview.html")
                elif str(mimetype).lower() in ["text/plain", "application/json"]:
                    response.content_type = 'application/json; charset="UTF-8"'
                    response.status_int = 200
                    response.status = "200 OK"
                    return simplejson.dumps(c.zipfile_contents)
                try:
                    mimetype = accept_list.pop(0)
                except IndexError:
                    mimetype = None
            # Whoops - nothing satisfies - return text/html
            return render("/zipfileview.html")
        elif http_method == "POST":
            params = request.POST
            #if not (params.has_key("filename") and params['filename']):
            #    abort(400, "You must supply a filename to unpack")

            if params.has_key("id") and params['id']:
                target_dataset_name = params['id']
            else:
                (head, fn) = os.path.split(path)
                (fn, ext) = os.path.splitext(fn)
                target_dataset_name = "%s-%s"%(id,fn)
            #target_dataset_name, current_dataset, post_filepath, silo, ident
            try:
                unpack_zip_item(target_dataset_name, dataset, path, rdfsilo, ident['repoze.who.userid'])
            except BadZipfile:
                abort(400, "Couldn't unpack zipfile")

            # conneg return
            accept_list = None
            if 'HTTP_ACCEPT' in request.environ:
                accept_list = conneg_parse(request.environ['HTTP_ACCEPT'])
            if not accept_list:
                accept_list= [MT("text", "html")]
            mimetype = accept_list.pop(0)
            while(mimetype):
                if str(mimetype).lower() in ["text/html", "text/xhtml"]:
                    # probably a browser - redirect to newly created dataset
                    redirect_to(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name)
                elif str(mimetype).lower() in ["text/plain", "application/json"]:
                    response.content_type = "text/plain"
                    response.status_int = 201
                    response.status = "201 Created"
                    #new_item = rdfsilo.get_item(target_dataset_name)
                    #response.headers.add("Content-Location", new_item.uri)
                    return "Created"
                try:
                    mimetype = accept_list.pop(0)
                except IndexError:
                    mimetype = None
            # Whoops - nothing satisfies - return text/plain
            response.content_type = "text/plain"
            response.status_int = 201
            #new_item = rdfsilo.get_item(target_dataset_name)
            #response.headers.add("Content-Location", new_item.uri)
            response.status = "201 Created"
            return "Created"
<commit_msg>Forgot to import simplejson<commit_after>import logging
import os
import simplejson
from pylons import request, response, session, tmpl_context as c
from pylons.controllers.util import abort, redirect_to
from pylons import app_globals as ag

from rdfdatabank.lib.base import BaseController, render
from rdfdatabank.lib.utils import create_new
from rdfdatabank.lib.file_unpack import check_file_mimetype, BadZipfile, get_zipfiles_in_dataset, unpack_zip_item, read_zipfile
from rdfdatabank.lib.conneg import MimeType as MT, parse as conneg_parse

log = logging.getLogger(__name__)

class ItemsController(BaseController):
    def siloview(self, silo):
        abort(403, "Forbidden")

    def datasetview(self, silo, id):
        #tmpl_context variables needed: c.silo_name, c.zipfiles, c.ident, c.id, c.path

        c.silo_name = silo
        c.id = id

        if not request.environ.get('repoze.who.identity'):
            abort(401, "Not Authorised")

        ident = request.environ.get('repoze.who.identity')
        c.ident = ident
        granary_list = ag.granary.silos
        if ident:
            silos = ag.authz(granary_list, ident)
            if silo not in silos:
                abort(403, "Forbidden")
        else:
            abort(403, "Forbidden")

        rdfsilo = ag.granary.get_rdf_silo(silo)
        if not rdfsilo.exists(id):
            abort (403, "Forbidden")

        dataset = rdfsilo.get_item(id)

        http_method = request.environ['REQUEST_METHOD']
        if http_method == "GET":
            c.zipfiles = get_zipfiles_in_dataset(dataset)
            # conneg return
            accept_list = None
            if 'HTTP_ACCEPT' in request.environ:
                accept_list = conneg_parse(request.environ['HTTP_ACCEPT'])
            if not accept_list:
                accept_list= [MT("text", "html")]
            mimetype = accept_list.pop(0)
            while(mimetype):
                if str(mimetype).lower() in ["text/html", "text/xhtml"]:
                    return render("/list_of_zipfiles.html")
                elif str(mimetype).lower() in ["text/plain", "application/json"]:
                    response.content_type = 'application/json; charset="UTF-8"'
                    response.status_int = 200
                    response.status = "200 OK"
                    return simplejson.dumps(dict(c.zipfiles))
                try:
                    mimetype = accept_list.pop(0)
                except IndexError:
                    mimetype = None
            #Whoops nothing satisfies - return text/html
            return render("/list_of_zipfiles.html")
        elif http_method == "POST":
            params = request.POST
            if not (params.has_key("filename") and params['filename']):
                abort(400, "You must supply a filename to unpack")
            item_real_filepath = dataset.to_dirpath()
            target_filepath = "%s/%s"%(item_real_filepath, params['filename'])
            if not os.path.isfile(target_filepath):
                abort(404, "File to unpack not found")
            if not check_file_mimetype(target_filepath, 'application/zip'):
                abort(415, "File is not of type application/zip")

            if params.has_key("id") and params['id']:
                target_dataset_name = params['id']
            else:
                (head, fn) = os.path.split(params['filename'])
                (fn, ext) = os.path.splitext(fn)
                target_dataset_name = "%s-%s"%(id,fn)

            if not rdfsilo.exists(target_dataset_name):
                target_dataset = create_new(rdfsilo, target_dataset_name, ident['repoze.who.userid'])
            else:
                target_dataset = rdfsilo.get_item(target_dataset_name)

            try:
                unpack_zip_item(target_dataset, dataset, params['filename'], rdfsilo, ident['repoze.who.userid'])
            except BadZipfile:
                abort(400, "Couldn't unpack zipfile")

            # conneg return
            accept_list = None
            if 'HTTP_ACCEPT' in request.environ:
                accept_list = conneg_parse(request.environ['HTTP_ACCEPT'])
            if not accept_list:
                accept_list= [MT("text", "html")]
            mimetype = accept_list.pop(0)
            while(mimetype):
                if str(mimetype).lower() in ["text/html", "text/xhtml"]:
                    # probably a browser - redirect to newly created dataset
                    redirect_to(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name)
                elif str(mimetype).lower() in ["text/plain", "application/json"]:
                    response.content_type = "text/plain"
                    response.status_int = 201
                    response.status = "201 Created"
                    #new_item = rdfsilo.get_item(target_dataset_name)
                    #response.headers["Content-Location"] = new_item.uri
                    #response.headers.add("Content-Location", new_item.uri)
                    #response.content_location = item.uri
                    #response.headers['location'] = item.uri
                    #response.location = item.uri
                    return "Created"
                try:
                    mimetype = accept_list.pop(0)
                except IndexError:
                    mimetype = None
            # Whoops - nothing satisfies - return text/plain
            response.content_type = "text/plain"
            response.status_int = 201
            response.status = "201 Created"
            #new_item = rdfsilo.get_item(target_dataset_name)
            #response.headers.add("Content-Location", new_item.uri)
            return "Created"

    def itemview(self, silo, id, path):
        #tmpl_context variables needed: c.silo_name, c.zipfile_contents c.ident, c.id, c.path
        c.silo_name = silo
        c.id = id
        c.path = path

        if not request.environ.get('repoze.who.identity'):
            abort(401, "Not Authorised")

        if not path:
            abort(400, "You must supply a filename to unpack")

        ident = request.environ.get('repoze.who.identity')
        c.ident = ident
        granary_list = ag.granary.silos
        if ident:
            silos = ag.authz(granary_list, ident)
            if silo not in silos:
                abort(403, "Forbidden")
        else:
            abort(403, "Forbidden")

        rdfsilo = ag.granary.get_rdf_silo(silo)
        if not rdfsilo.exists(id):
            abort (403, "Forbidden")

        dataset = rdfsilo.get_item(id)
        item_real_filepath = dataset.to_dirpath()
        target_filepath = "%s/%s"%(item_real_filepath, path)
        #c.parts = dataset.list_parts(detailed=False)
        if not dataset.isfile(path):
            abort(404, "File not found")
        if not os.path.isfile(target_filepath):
            abort(404, "File not found")
        if not check_file_mimetype(target_filepath, 'application/zip'):
            abort(415, "File is not of type application/zip")

        http_method = request.environ['REQUEST_METHOD']
        if http_method == "GET":
            try:
                c.zipfile_contents = read_zipfile(target_filepath)
            except BadZipfile:
                abort(400, "Could not read zipfile")
            # conneg return
            accept_list = None
            if 'HTTP_ACCEPT' in request.environ:
                accept_list = conneg_parse(request.environ['HTTP_ACCEPT'])
            if not accept_list:
                accept_list= [MT("text", "html")]
            mimetype = accept_list.pop(0)
            while(mimetype):
                if str(mimetype).lower() in ["text/html", "text/xhtml"]:
                    return render("/zipfileview.html")
                elif str(mimetype).lower() in ["text/plain", "application/json"]:
                    response.content_type = 'application/json; charset="UTF-8"'
                    response.status_int = 200
                    response.status = "200 OK"
                    return simplejson.dumps(c.zipfile_contents)
                try:
                    mimetype = accept_list.pop(0)
                except IndexError:
                    mimetype = None
            # Whoops - nothing satisfies - return text/html
            return render("/zipfileview.html")
        elif http_method == "POST":
            params = request.POST
            #if not (params.has_key("filename") and params['filename']):
            #    abort(400, "You must supply a filename to unpack")

            if params.has_key("id") and params['id']:
                target_dataset_name = params['id']
            else:
                (head, fn) = os.path.split(path)
                (fn, ext) = os.path.splitext(fn)
                target_dataset_name = "%s-%s"%(id,fn)
            #target_dataset_name, current_dataset, post_filepath, silo, ident
            try:
                unpack_zip_item(target_dataset_name, dataset, path, rdfsilo, ident['repoze.who.userid'])
            except BadZipfile:
                abort(400, "Couldn't unpack zipfile")

            # conneg return
            accept_list = None
            if 'HTTP_ACCEPT' in request.environ:
                accept_list = conneg_parse(request.environ['HTTP_ACCEPT'])
            if not accept_list:
                accept_list= [MT("text", "html")]
            mimetype = accept_list.pop(0)
            while(mimetype):
                if str(mimetype).lower() in ["text/html", "text/xhtml"]:
                    # probably a browser - redirect to newly created dataset
                    redirect_to(controller="datasets", action="datasetview", silo=silo, id=target_dataset_name)
                elif str(mimetype).lower() in ["text/plain", "application/json"]:
                    response.content_type = "text/plain"
                    response.status_int = 201
                    response.status = "201 Created"
                    #new_item = rdfsilo.get_item(target_dataset_name)
                    #response.headers.add("Content-Location", new_item.uri)
                    return "Created"
                try:
                    mimetype = accept_list.pop(0)
                except IndexError:
                    mimetype = None
            # Whoops - nothing satisfies - return text/plain
            response.content_type = "text/plain"
            response.status_int = 201
            #new_item = rdfsilo.get_item(target_dataset_name)
            #response.headers.add("Content-Location", new_item.uri)
            response.status = "201 Created"
            return "Created"
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Make a big square map of the ARM processor's view of memory. Each pixel will
# represent 256 bytes of memory. 16 million pixels and we have the whole
# 32-bit address space. We can fit this in a 4096x4096 image. Maybe some neat
# patterns emerge. It's kinda slow! Maybe it will crash!

__all__ = ['categorize_block', 'hilbert_curve', 'categorize_block_array', 'memsquare']

from dump import *
from math import log
from hilbert import hilbert
import remote, sys, png, struct, time

def categorize_block(d, address, size):
    # Look at everything we can get in one round trip.
    # Returns scaled red and green values, but raw blue values;
    # they can't be calculated until the whole image is known.

    t1 = time.time()
    head = read_block(d, address, 0x1d)
    t2 = time.time()

    # Red/green channels: Statistics. Red is mean,
    # green is mod-256 sum of squared differences between
    # consecutive bytes. Blue encodes the access time.
    # The amount of time it takes to read is a good
    # indicator of the bus type or cache settings!

    red = 0
    green = 0
    prev = '\x00'

    for b in head:
        signed_diff = struct.unpack('b', struct.pack('B', 0xFF & (ord(b) - ord(prev))))[0]
        red += ord(b)
        green += signed_diff * signed_diff
        prev = b

    return [
        min(0xFF, max(0, int(red / len(head)))),   # Divide by total to finish calculating the mean
        green & 0xFF,                              # Modulo diff from above
        t2 - t1                                    # Blue is in floating point seconds. We post-process below.
    ]

def categorize_block_array(d, base_address = 0, blocksize = 0x100, pixelsize = 4096):
    print 'Categorizing blocks in memory, following a 2D Hilbert curve'

    a = []
    timestamp = time.time()
    first_time = timestamp

    for y in xrange(pixelsize):
        row = []
        for x in xrange(pixelsize):
            addr = base_address + blocksize * hilbert(x, y, pixelsize)
            row.extend(categorize_block(d, addr, blocksize))

            # Keep the crowd informed!
            now = time.time()
            if now > timestamp + 0.5:

                # Estimate time
                completion = (x + y*pixelsize) / float(pixelsize*pixelsize)
                elapsed = now - first_time
                if completion > 0.00001:
                    remaining = elapsed / completion
                else:
                    # Placeholder; not complete enough to estimate
                    remaining = 0

                print 'block %08x - (%4d,%4d) of %d, %6.2f%% -- %2d:%02d:%02d elapsed, %02d:%02d:%02d est. remaining' % (
                    addr, x, y, pixelsize, 100 * completion,
                    elapsed / (60 * 60), (elapsed / 60) % 60, elapsed % 60,
                    remaining / (60 * 60), (remaining / 60) % 60, remaining % 60)
                timestamp = now

        a.append(row)

    print 'Calculating global scaling for blue channel'

    # The blue channel still has raw time deltas. Calculate pecentiles so we can scale them.

    times = []
    for row in a:
        for x in xrange(pixelsize):
            times.append(row[2 + 3*x])
    times.sort()

    percentile_low  = log(times[int( len(times) * 0.05 )])
    percentile_high = log(times[int( len(times) * 0.95 )])
    percentile_s    = 256.0 / (percentile_high - percentile_low)

    for row in a:
        for x in xrange(pixelsize):
            row[2+3*x] = min(255, max(0, int(0.5 + (log(row[2+3*x]) - percentile_low) * percentile_s)))

    print 'Done scaling'
    return a

def memsquare(d, filename = 'memsquare.png', base_address = 0, blocksize = 0x100, pixelsize = 4096):
    b = categorize_block_array(d, base_address, blocksize, pixelsize)
    w = png.Writer(len(b[0])/3, len(b))
    f = open(filename, 'wb')
    w.write(f, b)
    f.close()
    print 'Wrote %s' % filename

if __name__ == "__main__":
    if len(sys.argv) == 1:
        memsquare(remote.Device())
    elif len(sys.argv) > 2 or sys.argv[1][0] == '-':
        print "usage: %s [filename]" % sys.argv[0]
        sys.exit(1)
    else:
        memsquare(remote.Device(), sys.argv[1])
<commit_msg>Fix __all__<commit_after>#!/usr/bin/env python

# Make a big square map of the ARM processor's view of memory. Each pixel will
# represent 256 bytes of memory. 16 million pixels and we have the whole
# 32-bit address space. We can fit this in a 4096x4096 image. Maybe some neat
# patterns emerge. It's kinda slow! Maybe it will crash!

__all__ = ['categorize_block', 'categorize_block_array', 'memsquare']

from dump import *
from math import log
from hilbert import hilbert
import remote, sys, png, struct, time

def categorize_block(d, address, size):
    # Look at everything we can get in one round trip.
    # Returns scaled red and green values, but raw blue values;
    # they can't be calculated until the whole image is known.

    t1 = time.time()
    head = read_block(d, address, 0x1d)
    t2 = time.time()

    # Red/green channels: Statistics. Red is mean,
    # green is mod-256 sum of squared differences between
    # consecutive bytes. Blue encodes the access time.
    # The amount of time it takes to read is a good
    # indicator of the bus type or cache settings!

    red = 0
    green = 0
    prev = '\x00'

    for b in head:
        signed_diff = struct.unpack('b', struct.pack('B', 0xFF & (ord(b) - ord(prev))))[0]
        red += ord(b)
        green += signed_diff * signed_diff
        prev = b

    return [
        min(0xFF, max(0, int(red / len(head)))),   # Divide by total to finish calculating the mean
        green & 0xFF,                              # Modulo diff from above
        t2 - t1                                    # Blue is in floating point seconds. We post-process below.
    ]

def categorize_block_array(d, base_address = 0, blocksize = 0x100, pixelsize = 4096):
    print 'Categorizing blocks in memory, following a 2D Hilbert curve'

    a = []
    timestamp = time.time()
    first_time = timestamp

    for y in xrange(pixelsize):
        row = []
        for x in xrange(pixelsize):
            addr = base_address + blocksize * hilbert(x, y, pixelsize)
            row.extend(categorize_block(d, addr, blocksize))

            # Keep the crowd informed!
            now = time.time()
            if now > timestamp + 0.5:

                # Estimate time
                completion = (x + y*pixelsize) / float(pixelsize*pixelsize)
                elapsed = now - first_time
                if completion > 0.00001:
                    remaining = elapsed / completion
                else:
                    # Placeholder; not complete enough to estimate
                    remaining = 0

                print 'block %08x - (%4d,%4d) of %d, %6.2f%% -- %2d:%02d:%02d elapsed, %02d:%02d:%02d est. remaining' % (
                    addr, x, y, pixelsize, 100 * completion,
                    elapsed / (60 * 60), (elapsed / 60) % 60, elapsed % 60,
                    remaining / (60 * 60), (remaining / 60) % 60, remaining % 60)
                timestamp = now

        a.append(row)

    print 'Calculating global scaling for blue channel'

    # The blue channel still has raw time deltas. Calculate pecentiles so we can scale them.

    times = []
    for row in a:
        for x in xrange(pixelsize):
            times.append(row[2 + 3*x])
    times.sort()

    percentile_low  = log(times[int( len(times) * 0.05 )])
    percentile_high = log(times[int( len(times) * 0.95 )])
    percentile_s    = 256.0 / (percentile_high - percentile_low)

    for row in a:
        for x in xrange(pixelsize):
            row[2+3*x] = min(255, max(0, int(0.5 + (log(row[2+3*x]) - percentile_low) * percentile_s)))

    print 'Done scaling'
    return a

def memsquare(d, filename = 'memsquare.png', base_address = 0, blocksize = 0x100, pixelsize = 4096):
    b = categorize_block_array(d, base_address, blocksize, pixelsize)
    w = png.Writer(len(b[0])/3, len(b))
    f = open(filename, 'wb')
    w.write(f, b)
    f.close()
    print 'Wrote %s' % filename

if __name__ == "__main__":
    if len(sys.argv) == 1:
        memsquare(remote.Device())
    elif len(sys.argv) > 2 or sys.argv[1][0] == '-':
        print "usage: %s [filename]" % sys.argv[0]
        sys.exit(1)
    else:
        memsquare(remote.Device(), sys.argv[1])
<|endoftext|>