<commit_before>#   Copyright 2008 Red Hat, Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import ConfigParser
import os
import socket
import syslog
import re
import tarfile
import zipfile
import time
try:
   from subprocess import *
   use_popen2 = False
except:
   from popen2 import *
   use_popen2 = True
from cStringIO import StringIO

SUCCESS = 0
FAILURE = 1

# Generic enumerated type
class wf_type:
   def __init__(self,list=[]):
      count = 0
      for item in list.split():
         vars(self)[item] = count
         count = count + 1

   def __setattr__(self, name, value):
      raise ValueError('Modifying values is not allowed')

   def __delattr__(self, name):
      raise ValueError('Deleting entries is not allowed')

# The type of requests from condor to act upon
condor_wf_types = wf_type('get_work prepare_job reply_claim_accept reply_claim_reject update_job_status exit_exit exit_remove exit_hold exit_evict')

# The information about the job
class condor_wf(object):
    def set_type(self, thetype):
           self.__type__ = thetype

    def get_type(self):
        return self.__type__

    type = property(get_type, set_type)

    def set_data(self, string):
           self.__data__ = string

    def get_data(self):
        return self.__data__

    data = property(get_data, set_data)

# General failure exception
class general_exception(Exception):
   def __init__(self, level, *msgs):
      self.level = level
      self.msgs = msgs

# Configuration exception
class config_err(Exception):
   def __init__(self, msg):
      self.msg = msg

def run_cmd(cmd, args):
   """Runs the command specified in 'cmd' with the given 'args' using
      either the subprocess module or the open2 module, depending on which
      is supported.  The subprocess module is favored.  Returns
      (return_code, stdout, stderr)"""
   retcode = -1
   std_out = ''
   std_err = ''
   if use_popen2 == False:
      obj = Popen([cmd, args], stdout=PIPE, stderr=PIPE)
      (std_out, std_err) = obj.communicate()
      retcode = obj.returncode
   else:
      obj = Popen3('%s %s' % (cmd, args), True)
      retcode = obj.wait()
      try:
         std_out = obj.fromchild.readlines()[0]
         obj.fromchild.close()
      except:
         pass
      try:
         obj.tochild.close()
      except:
         pass
      try:
         std_err = obj.childerr.readlines()[0]
         obj.childerr.close()
      except:
         pass
   return ([retcode, std_out, std_err])

def read_condor_config(subsys, attr_list):
   """ Uses condor_config_val to look up values in condor's configuration.
       First looks for subsys_param, then for the newer subsys.param.
       Returns map(param, value)"""
   config = {}
   for attr in attr_list:
      (rcode, value, sterr) = run_cmd('condor_config_val', '%s_%s' % (subsys, attr))
      if rcode == 0:
         config[attr.lower()] = value.rstrip().lstrip()
      else:
         # Try the newer <subsys>.param form
         (rcode, value, sterr) = run_cmd('condor_config_val', '%s.%s' % (subsys, attr))
         if rcode == 0:
            config[attr.lower()] = value.rstrip().lstrip()
         else:
            # Config value not found.  Raise an exception
            raise config_err('"%s_%s" is not defined' % (subsys, attr))
   return config

def read_config_file(config_file, section):
   """Given configuration file and section names, returns a list of all value
     	 pairs in the config file as a dictionary"""
   parser = ConfigParser.ConfigParser()

   # Parse the config file
   if os.path.exists(config_file) == False:
      raise config_err('%s configuration file does not exist.' % config_file)
   else:
      try:
         parser.readfp(open(config_file))
         items = parser.items(section)
      except Exception, error:
         raise config_err('Problems reading %s.' % config_file, str(error))

      # Take the list of lists and convert into a dictionary
      dict = {}
      for list in items:
         dict[list[0]] = list[1].rstrip().lstrip()
      return dict

def socket_read_all(sock):
   """Read all data waiting to be read on the given socket.  The first attempt
      to read will be a blocking call, and all subsequent reads will be
      non-blocking."""
   msg = ''
   old_timeout = sock.gettimeout()
   data = sock.recv(1024)
   sock.settimeout(0.1)
   try:
      while len(data) != 0:
         msg += data
         data = sock.recv(1024)
   except socket.timeout, timeout:
      pass
   except Exception, error:
      close_socket(sock)
      if error != None:
         raise general_exception(syslog.LOG_ERR, 'socket error %d: %s' % (error[0], error[1]))
   sock.settimeout(old_timeout)
   return msg

def close_socket(connection):
   """Performs a shutdown and a close on the socket.  Any errors
      are logged to the system logging service.  Raises a general_exception
      if any errors are encountered"""
   try:
      try:
         # python 2.3 doesn't have SHUT_WR defined, so use it's value (1)
         connection.shutdown(1)
      except Exception, error:
         if error != None:
            raise general_exception(syslog.LOG_ERR, 'socket error %d: %s' % (error[0], error[1]))
      try:
         data = connection.recv(4096)
         while len(data) != 0:
            data = connection.recv(4096)
      except:
         pass
   finally:
      connection.close()
      connection = None

def log_messages(exception):
   """Logs messages in the passed general_exception exception to the
      system logger"""
   for msg in exception.msgs:
      if (msg != ''):
         syslog.syslog(exception.level, msg)

def write_file(filename, data):
   """Writes the given data into the given filename in pieces to account for
      large files"""
   file_ptr = open(filename, 'wb')
   file_str = StringIO(data)
   buffer_length = 2 ** 20
   contents = file_str.read(buffer_length)
   while contents:
      file_ptr.write(contents)
      contents = file_str.read(buffer_length)
   file_ptr.flush()
   file_ptr.close()

def grep(pattern, data):
   """Returns the first instance found of pattern in data.  If pattern
      doesn't exist in data, None is returned.  If data contains groups,
      returns the matching subgroups instead"""
   for line in data.split('\n'):
      match = re.match(pattern, line)
      if match != None:
         break
   if match != None and match.groups() != None:
      found = match.groups()
   else:
      found = match
   return(found)

def zip_extract(filename):
   """Unzips the contents of the filename given, preserving permissions and
      ownership"""
   zip = zipfile.ZipFile(filename, 'r')
   contents = zip.namelist()

   # Loop through the archive names and extract the contents
   # Make sure to create higher level directories before creating a lower
   # level file or directory (if applicable)
   contents.sort()
   for item in contents:
      if item.endswith('/'):
         # This item is all directories, so recursively create them if
         # they don't already exist
         if not os.path.exists(item):
            os.makedirs(item)
      else:
         # This is a directory + file or a file by itself, so create the
         # directory hierarchy first (if applicable) then extract the file.
         dirs = item.split('/')
         dir_structure = ''
         for count in range(0,len(dirs)-1):
            full_path = os.path.join(dir_structure, dirs[count])
            if dirs[count] != '' and not os.path.exists(full_path):
               os.mkdir(full_path)
            dir_structure = full_path

         write_file(item, zip.read(item))

   # Set the perserved permissions and timestamp for the extracted
   # files/directories
   for name in contents:
      info = zip.getinfo(name)
      file_time = time.mktime(info.date_time + (0, 0, -1))
      os.chmod(name, info.external_attr >> 16L)
      os.utime(name, (file_time, file_time))

def tarball_extract(filename):
   """Extracts the tarball (.tar.gz) given"""
   tarball = tarfile.open(filename, 'r:gz')
   for file in tarball.getnames():
      tarball.extract(file)
   tarball.close()
<commit_msg>Fixed handling of multiple args, space delimited, when using Popen in run_cmd<commit_after>#   Copyright 2008 Red Hat, Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import ConfigParser
import os
import socket
import syslog
import re
import tarfile
import zipfile
import time
try:
   from subprocess import *
   use_popen2 = False
except:
   from popen2 import *
   use_popen2 = True
from cStringIO import StringIO

SUCCESS = 0
FAILURE = 1

# Generic enumerated type
class wf_type:
   def __init__(self,list=[]):
      count = 0
      for item in list.split():
         vars(self)[item] = count
         count = count + 1

   def __setattr__(self, name, value):
      raise ValueError('Modifying values is not allowed')

   def __delattr__(self, name):
      raise ValueError('Deleting entries is not allowed')

# The type of requests from condor to act upon
condor_wf_types = wf_type('get_work prepare_job reply_claim_accept reply_claim_reject update_job_status exit_exit exit_remove exit_hold exit_evict')

# The information about the job
class condor_wf(object):
    def set_type(self, thetype):
           self.__type__ = thetype

    def get_type(self):
        return self.__type__

    type = property(get_type, set_type)

    def set_data(self, string):
           self.__data__ = string

    def get_data(self):
        return self.__data__

    data = property(get_data, set_data)

# General failure exception
class general_exception(Exception):
   def __init__(self, level, *msgs):
      self.level = level
      self.msgs = msgs

# Configuration exception
class config_err(Exception):
   def __init__(self, msg):
      self.msg = msg

def run_cmd(cmd, args):
   """Runs the command specified in 'cmd' with the given 'args' using
      either the subprocess module or the open2 module, depending on which
      is supported.  The subprocess module is favored.  Returns
      (return_code, stdout, stderr)"""
   retcode = -1
   std_out = ''
   std_err = ''
   if use_popen2 == False:
      cmd_list = [cmd]
      cmd_list += args.split(' ')
      obj = Popen(cmd_list, stdout=PIPE, stderr=PIPE)
      (std_out, std_err) = obj.communicate()
      retcode = obj.returncode
   else:
      obj = Popen3('%s %s' % (cmd, args), True)
      retcode = obj.wait()
      try:
         std_out = obj.fromchild.readlines()[0]
         obj.fromchild.close()
      except:
         pass
      try:
         obj.tochild.close()
      except:
         pass
      try:
         std_err = obj.childerr.readlines()[0]
         obj.childerr.close()
      except:
         pass
   return ([retcode, std_out, std_err])

def read_condor_config(subsys, attr_list):
   """ Uses condor_config_val to look up values in condor's configuration.
       First looks for subsys_param, then for the newer subsys.param.
       Returns map(param, value)"""
   config = {}
   for attr in attr_list:
      (rcode, value, sterr) = run_cmd('condor_config_val', '%s_%s' % (subsys, attr))
      if rcode == 0:
         config[attr.lower()] = value.rstrip().lstrip()
      else:
         # Try the newer <subsys>.param form
         (rcode, value, sterr) = run_cmd('condor_config_val', '%s.%s' % (subsys, attr))
         if rcode == 0:
            config[attr.lower()] = value.rstrip().lstrip()
         else:
            # Config value not found.  Raise an exception
            raise config_err('"%s_%s" is not defined' % (subsys, attr))
   return config

def read_config_file(config_file, section):
   """Given configuration file and section names, returns a list of all value
     	 pairs in the config file as a dictionary"""
   parser = ConfigParser.ConfigParser()

   # Parse the config file
   if os.path.exists(config_file) == False:
      raise config_err('%s configuration file does not exist.' % config_file)
   else:
      try:
         parser.readfp(open(config_file))
         items = parser.items(section)
      except Exception, error:
         raise config_err('Problems reading %s.' % config_file, str(error))

      # Take the list of lists and convert into a dictionary
      dict = {}
      for list in items:
         dict[list[0]] = list[1].rstrip().lstrip()
      return dict

def socket_read_all(sock):
   """Read all data waiting to be read on the given socket.  The first attempt
      to read will be a blocking call, and all subsequent reads will be
      non-blocking."""
   msg = ''
   old_timeout = sock.gettimeout()
   data = sock.recv(1024)
   sock.settimeout(0.1)
   try:
      while len(data) != 0:
         msg += data
         data = sock.recv(1024)
   except socket.timeout, timeout:
      pass
   except Exception, error:
      close_socket(sock)
      if error != None:
         raise general_exception(syslog.LOG_ERR, 'socket error %d: %s' % (error[0], error[1]))
   sock.settimeout(old_timeout)
   return msg

def close_socket(connection):
   """Performs a shutdown and a close on the socket.  Any errors
      are logged to the system logging service.  Raises a general_exception
      if any errors are encountered"""
   try:
      try:
         # python 2.3 doesn't have SHUT_WR defined, so use it's value (1)
         connection.shutdown(1)
      except Exception, error:
         if error != None:
            raise general_exception(syslog.LOG_ERR, 'socket error %d: %s' % (error[0], error[1]))
      try:
         data = connection.recv(4096)
         while len(data) != 0:
            data = connection.recv(4096)
      except:
         pass
   finally:
      connection.close()
      connection = None

def log_messages(exception):
   """Logs messages in the passed general_exception exception to the
      system logger"""
   for msg in exception.msgs:
      if (msg != ''):
         syslog.syslog(exception.level, msg)

def write_file(filename, data):
   """Writes the given data into the given filename in pieces to account for
      large files"""
   file_ptr = open(filename, 'wb')
   file_str = StringIO(data)
   buffer_length = 2 ** 20
   contents = file_str.read(buffer_length)
   while contents:
      file_ptr.write(contents)
      contents = file_str.read(buffer_length)
   file_ptr.flush()
   file_ptr.close()

def grep(pattern, data):
   """Returns the first instance found of pattern in data.  If pattern
      doesn't exist in data, None is returned.  If data contains groups,
      returns the matching subgroups instead"""
   for line in data.split('\n'):
      match = re.match(pattern, line)
      if match != None:
         break
   if match != None and match.groups() != None:
      found = match.groups()
   else:
      found = match
   return(found)

def zip_extract(filename):
   """Unzips the contents of the filename given, preserving permissions and
      ownership"""
   zip = zipfile.ZipFile(filename, 'r')
   contents = zip.namelist()

   # Loop through the archive names and extract the contents
   # Make sure to create higher level directories before creating a lower
   # level file or directory (if applicable)
   contents.sort()
   for item in contents:
      if item.endswith('/'):
         # This item is all directories, so recursively create them if
         # they don't already exist
         if not os.path.exists(item):
            os.makedirs(item)
      else:
         # This is a directory + file or a file by itself, so create the
         # directory hierarchy first (if applicable) then extract the file.
         dirs = item.split('/')
         dir_structure = ''
         for count in range(0,len(dirs)-1):
            full_path = os.path.join(dir_structure, dirs[count])
            if dirs[count] != '' and not os.path.exists(full_path):
               os.mkdir(full_path)
            dir_structure = full_path

         write_file(item, zip.read(item))

   # Set the perserved permissions and timestamp for the extracted
   # files/directories
   for name in contents:
      info = zip.getinfo(name)
      file_time = time.mktime(info.date_time + (0, 0, -1))
      os.chmod(name, info.external_attr >> 16L)
      os.utime(name, (file_time, file_time))

def tarball_extract(filename):
   """Extracts the tarball (.tar.gz) given"""
   tarball = tarfile.open(filename, 'r:gz')
   for file in tarball.getnames():
      tarball.extract(file)
   tarball.close()
<|endoftext|>


<commit_before># coding=utf-8
import logging

from django.db.models.aggregates import Count
from django.http.response import HttpResponseRedirect
from django.views.generic.base import TemplateView
from django.core.urlresolvers import reverse
from organizations.models import Facility
from places.models import Region

logger = logging.getLogger(__name__)

class HomeView(TemplateView):
    template_name = "base_non_logged_in.html"

    def get(self, request, *args, **kwargs):
        if self.request.user.is_authenticated():
            return HttpResponseRedirect(reverse('helpdesk'))
        context = self.get_context_data(**kwargs)
        return self.render_to_response(context)

    def get_context_data(self, **kwargs):
        context = super(HomeView, self).get_context_data(**kwargs)

        context['regions'] = Region.objects.annotate(
            facility_count=Count('areas__places__facilities')).exclude(
            facility_count=0).prefetch_related('areas', 'areas__region').all()
        context['facilities'] = Facility.objects.select_related('place',
                                                                'place__area',
                                                                'place__area__region').all()
        return context
<commit_msg>Ordering of facilities on landing page (fixes #280)<commit_after># coding=utf-8
import logging

from django.db.models.aggregates import Count
from django.http.response import HttpResponseRedirect
from django.views.generic.base import TemplateView
from django.core.urlresolvers import reverse
from organizations.models import Facility
from places.models import Region

logger = logging.getLogger(__name__)

class HomeView(TemplateView):
    template_name = "base_non_logged_in.html"

    def get(self, request, *args, **kwargs):
        if self.request.user.is_authenticated():
            return HttpResponseRedirect(reverse('helpdesk'))
        context = self.get_context_data(**kwargs)
        return self.render_to_response(context)

    def get_context_data(self, **kwargs):
        context = super(HomeView, self).get_context_data(**kwargs)

        context['regions'] = Region.objects.annotate(
            facility_count=Count('areas__places__facilities')).exclude(
            facility_count=0).prefetch_related('areas', 'areas__region').all()
        context['facilities'] = Facility.objects.select_related('place',
                                                                'place__area',
                                                                'place__area__region').order_by('place').all()
        return context
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
#    Copyright 2010 OpenStack LLC.
#    Copyright 2010 United States Government as represented by the
#    Administrator of the National Aeronautics and Space Administration.
#    All Rights Reserved.
#
#    Copyright 2009 Facebook
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Implementation of an S3-like storage server based on local files.

Useful to test features that will eventually run on S3, or if you want to
run something locally that was once running on S3.

We don't support all the features of S3, but it does work with the
standard S3 client for the most basic semantics. To use the standard
S3 client with this module::

    c = S3.AWSAuthConnection("", "", server="localhost", port=8888,
                             is_secure=False)
    c.create_bucket("mybucket")
    c.put("mybucket", "mykey", "a value")
    print c.get("mybucket", "mykey").body

"""

import datetime
import json
import logging
import multiprocessing
import os
import urllib

from tornado import escape
from twisted.application import internet
from twisted.application import service
from twisted.web import error
from twisted.web import resource
from twisted.web import server
from twisted.web import static

from nova import context
from nova import exception
from nova import flags
from nova.auth import manager
from nova.objectstore import bucket
from nova.objectstore import image

FLAGS = flags.FLAGS

def render_xml(request, value):
    """Writes value as XML string to request"""
    assert isinstance(value, dict) and len(value) == 1
    request.setHeader("Content-Type", "application/xml; charset=UTF-8")

    name = value.keys()[0]
    request.write('<?xml version="1.0" encoding="UTF-8"?>\n')
    request.write('<' + escape.utf8(name) +
                 ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">')
    _render_parts(value.values()[0], request.write)
    request.write('</' + escape.utf8(name) + '>')
    request.finish()

def finish(request, content=None):
    """Finalizer method for request"""
    if content:
        request.write(content)
    request.finish()

def _render_parts(value, write_cb):
    """Helper method to render different Python objects to XML"""
    if isinstance(value, basestring):
        write_cb(escape.xhtml_escape(value))
    elif isinstance(value, int) or isinstance(value, long):
        write_cb(str(value))
    elif isinstance(value, datetime.datetime):
        write_cb(value.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
    elif isinstance(value, dict):
        for name, subvalue in value.iteritems():
            if not isinstance(subvalue, list):
                subvalue = [subvalue]
            for subsubvalue in subvalue:
                write_cb('<' + escape.utf8(name) + '>')
                _render_parts(subsubvalue, write_cb)
                write_cb('</' + escape.utf8(name) + '>')
    else:
        raise Exception("Unknown S3 value type %r", value)

def get_argument(request, key, default_value):
    """Returns the request's value at key, or default_value
    if not found
    """
    if key in request.args:
        return request.args[key][0]
    return default_value

def get_context(request):
    """Returns the supplied request's context object"""
    try:
        # Authorization Header format: 'AWS <access>:<secret>'
        authorization_header = request.getHeader('Authorization')
        if not authorization_header:
            raise exception.NotAuthorized
        auth_header_value = authorization_header.split(' ')[1]
        access, _ignored, secret = auth_header_value.rpartition(':')
        am = manager.AuthManager()
        (user, project) = am.authenticate(access,
                                          secret,
                                          {},
                                          request.method,
                                          request.getRequestHostname(),
                                          request.uri,
                                          headers=request.getAllHeaders(),
                                          check_type='s3')
        return context.RequestContext(user, project)
    except exception.Error as ex:
        logging.debug("Authentication Failure: %s", ex)
        raise exception.NotAuthorized

class ErrorHandlingResource(resource.Resource):
    """Maps exceptions to 404 / 401 codes.  Won't work for
    exceptions thrown after NOT_DONE_YET is returned.
    """
    # TODO(unassigned) (calling-all-twisted-experts): This needs to be
    #                   plugged in to the right place in twisted...
    #                   This doesn't look like it's the right place
    #                   (consider exceptions in getChild; or after
    #                   NOT_DONE_YET is returned
    def render(self, request):
        """Renders the response as XML"""
        try:
            return resource.Resource.render(self, request)
        except exception.NotFound:
            request.setResponseCode(404)
            return ''
        except exception.NotAuthorized:
            request.setResponseCode(403)
            return ''

class S3(ErrorHandlingResource):
    """Implementation of an S3-like storage server based on local files."""
    def __init__(self):
        ErrorHandlingResource.__init__(self)

    def getChild(self, name, request): # pylint: disable-msg=C0103
        """Returns either the image or bucket resource"""
        request.context = get_context(request)
        if name == '':
            return self
        elif name == '_images':
            return ImagesResource()
        else:
            return BucketResource(name)

    def render_GET(self, request): # pylint: disable-msg=R0201
        """Renders the GET request for a list of buckets as XML"""
        logging.debug('List of buckets requested')
        buckets = [b for b in bucket.Bucket.all() \
                   if b.is_authorized(request.context)]

        render_xml(request, {"ListAllMyBucketsResult": {
            "Buckets": {"Bucket": [b.metadata for b in buckets]},
        }})
        return server.NOT_DONE_YET

class BucketResource(ErrorHandlingResource):
    """A web resource containing an S3-like bucket"""
    def __init__(self, name):
        resource.Resource.__init__(self)
        self.name = name

    def getChild(self, name, request):
        """Returns the bucket resource itself, or the object resource
        the bucket contains if a name is supplied
        """
        if name == '':
            return self
        else:
            return ObjectResource(bucket.Bucket(self.name), name)

    def render_GET(self, request):
        "Returns the keys for the bucket resource"""
        logging.debug("List keys for bucket %s", self.name)

        try:
            bucket_object = bucket.Bucket(self.name)
        except exception.NotFound:
            return error.NoResource(message="No such bucket").render(request)

        if not bucket_object.is_authorized(request.context):
            raise exception.NotAuthorized

        prefix = get_argument(request, "prefix", u"")
        marker = get_argument(request, "marker", u"")
        max_keys = int(get_argument(request, "max-keys", 1000))
        terse = int(get_argument(request, "terse", 0))

        results = bucket_object.list_keys(prefix=prefix,
                                          marker=marker,
                                          max_keys=max_keys,
                                          terse=terse)
        render_xml(request, {"ListBucketResult": results})
        return server.NOT_DONE_YET

    def render_PUT(self, request):
        "Creates the bucket resource"""
        logging.debug("Creating bucket %s", self.name)
        logging.debug("calling bucket.Bucket.create(%r, %r)",
                      self.name,
                      request.context)
        bucket.Bucket.create(self.name, request.context)
        request.finish()
        return server.NOT_DONE_YET

    def render_DELETE(self, request):
        """Deletes the bucket resource"""
        logging.debug("Deleting bucket %s", self.name)
        bucket_object = bucket.Bucket(self.name)

        if not bucket_object.is_authorized(request.context):
            raise exception.NotAuthorized

        bucket_object.delete()
        request.setResponseCode(204)
        return ''

class ObjectResource(ErrorHandlingResource):
    """The resource returned from a bucket"""
    def __init__(self, bucket, name):
        resource.Resource.__init__(self)
        self.bucket = bucket
        self.name = name

    def render_GET(self, request):
        """Returns the object

        Raises NotAuthorized if user in request context is not
        authorized to delete the object.
        """
        logging.debug("Getting object: %s / %s", self.bucket.name, self.name)

        if not self.bucket.is_authorized(request.context):
            raise exception.NotAuthorized

        obj = self.bucket[urllib.unquote(self.name)]
        request.setHeader("Content-Type", "application/unknown")
        request.setHeader("Last-Modified",
                          datetime.datetime.utcfromtimestamp(obj.mtime))
        request.setHeader("Etag", '"' + obj.md5 + '"')
        return static.File(obj.path).render_GET(request)

    def render_PUT(self, request):
        """Modifies/inserts the object and returns a result code

        Raises NotAuthorized if user in request context is not
        authorized to delete the object.
        """
        logging.debug("Putting object: %s / %s", self.bucket.name, self.name)

        if not self.bucket.is_authorized(request.context):
            raise exception.NotAuthorized

        key = urllib.unquote(self.name)
        request.content.seek(0, 0)
        self.bucket[key] = request.content.read()
        request.setHeader("Etag", '"' + self.bucket[key].md5 + '"')
        finish(request)
        return server.NOT_DONE_YET

    def render_DELETE(self, request):
        """Deletes the object and returns a result code

        Raises NotAuthorized if user in request context is not
        authorized to delete the object.
        """

        logging.debug("Deleting object: %s / %s",
                      self.bucket.name,
                      self.name)

        if not self.bucket.is_authorized(request.context):
            raise exception.NotAuthorized

        del self.bucket[urllib.unquote(self.name)]
        request.setResponseCode(204)
        return ''

class ImageResource(ErrorHandlingResource):
    """A web resource representing a single image"""
    isLeaf = True

    def __init__(self, name):
        resource.Resource.__init__(self)
        self.img = image.Image(name)

    def render_GET(self, request):
        """Returns the image file"""
        if not self.img.is_authorized(request.context, True):
            raise exception.NotAuthorized
        return static.File(self.img.image_path,
                           defaultType='application/octet-stream'
                          ).render_GET(request)

class ImagesResource(resource.Resource):
    """A web resource representing a list of images"""
    def getChild(self, name, _request):
        """Returns itself or an ImageResource if no name given"""
        if name == '':
            return self
        else:
            return ImageResource(name)

    def render_GET(self, request): # pylint: disable-msg=R0201
        """ returns a json listing of all images
            that a user has permissions to see """

        images = [i for i in image.Image.all() \
                  if i.is_authorized(request.context, readonly=True)]

        # Bug #617776:
        # We used to have 'type' in the image metadata, but this field
        # should be called 'imageType', as per the EC2 specification.
        # For compat with old metadata files we copy type to imageType if
        # imageType is not present.
        # For compat with euca2ools (and any other clients using the
        # incorrect name) we copy imageType to type.
        # imageType is primary if we end up with both in the metadata file
        # (which should never happen).
        def decorate(m):
            if 'imageType' not in m and 'type' in m:
                m[u'imageType'] = m['type']
            elif 'imageType' in m:
                m[u'type'] = m['imageType']
            if 'displayName' not in m:
                m[u'displayName'] = u''
            return m

        request.write(json.dumps([decorate(i.metadata) for i in images]))
        request.finish()
        return server.NOT_DONE_YET

    def render_PUT(self, request): # pylint: disable-msg=R0201
        """ create a new registered image """

        image_id = get_argument(request, 'image_id', u'')
        image_location = get_argument(request, 'image_location', u'')

        image_path = os.path.join(FLAGS.images_path, image_id)
        if not image_path.startswith(FLAGS.images_path) or \
           os.path.exists(image_path):
            raise exception.NotAuthorized

        bucket_object = bucket.Bucket(image_location.split("/")[0])

        if not bucket_object.is_authorized(request.context):
            raise exception.NotAuthorized

        p = multiprocessing.Process(target=image.Image.register_aws_image,
                args=(image_id, image_location, request.context))
        p.start()
        return ''

    def render_POST(self, request): # pylint: disable-msg=R0201
        """Update image attributes: public/private"""

        # image_id required for all requests
        image_id = get_argument(request, 'image_id', u'')
        image_object = image.Image(image_id)
        if not image_object.is_authorized(request.context):
            logging.debug("not authorized for render_POST in images")
            raise exception.NotAuthorized

        operation = get_argument(request, 'operation', u'')
        if operation:
            # operation implies publicity toggle
            logging.debug("handling publicity toggle")
            image_object.set_public(operation=='add')
        else:
            # other attributes imply update
            logging.debug("update user fields")
            clean_args = {}
            for arg in request.args.keys():
                clean_args[arg] = request.args[arg][0]
            image_object.update_user_editable_fields(clean_args)
        return ''

    def render_DELETE(self, request): # pylint: disable-msg=R0201
        """Delete a registered image"""
        image_id = get_argument(request, "image_id", u"")
        image_object = image.Image(image_id)

        if not image_object.is_authorized(request.context):
            raise exception.NotAuthorized

        image_object.delete()

        request.setResponseCode(204)
        return ''

def get_site():
    """Support for WSGI-like interfaces"""
    root = S3()
    site = server.Site(root)
    return site

def get_application():
    """Support WSGI-like interfaces"""
    factory = get_site()
    application = service.Application("objectstore")
    # Disabled because of lack of proper introspection in Twisted
    # or possibly different versions of twisted?
    # pylint: disable-msg=E1101
    objectStoreService = internet.TCPServer(FLAGS.s3_port, factory)
    objectStoreService.setServiceParent(application)
    return application
<commit_msg>Construct exception instead of raising a class.<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4
#
#    Copyright 2010 OpenStack LLC.
#    Copyright 2010 United States Government as represented by the
#    Administrator of the National Aeronautics and Space Administration.
#    All Rights Reserved.
#
#    Copyright 2009 Facebook
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Implementation of an S3-like storage server based on local files.

Useful to test features that will eventually run on S3, or if you want to
run something locally that was once running on S3.

We don't support all the features of S3, but it does work with the
standard S3 client for the most basic semantics. To use the standard
S3 client with this module::

    c = S3.AWSAuthConnection("", "", server="localhost", port=8888,
                             is_secure=False)
    c.create_bucket("mybucket")
    c.put("mybucket", "mykey", "a value")
    print c.get("mybucket", "mykey").body

"""

import datetime
import json
import logging
import multiprocessing
import os
import urllib

from tornado import escape
from twisted.application import internet
from twisted.application import service
from twisted.web import error
from twisted.web import resource
from twisted.web import server
from twisted.web import static

from nova import context
from nova import exception
from nova import flags
from nova.auth import manager
from nova.objectstore import bucket
from nova.objectstore import image

FLAGS = flags.FLAGS

def render_xml(request, value):
    """Writes value as XML string to request"""
    assert isinstance(value, dict) and len(value) == 1
    request.setHeader("Content-Type", "application/xml; charset=UTF-8")

    name = value.keys()[0]
    request.write('<?xml version="1.0" encoding="UTF-8"?>\n')
    request.write('<' + escape.utf8(name) +
                 ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">')
    _render_parts(value.values()[0], request.write)
    request.write('</' + escape.utf8(name) + '>')
    request.finish()

def finish(request, content=None):
    """Finalizer method for request"""
    if content:
        request.write(content)
    request.finish()

def _render_parts(value, write_cb):
    """Helper method to render different Python objects to XML"""
    if isinstance(value, basestring):
        write_cb(escape.xhtml_escape(value))
    elif isinstance(value, int) or isinstance(value, long):
        write_cb(str(value))
    elif isinstance(value, datetime.datetime):
        write_cb(value.strftime("%Y-%m-%dT%H:%M:%S.000Z"))
    elif isinstance(value, dict):
        for name, subvalue in value.iteritems():
            if not isinstance(subvalue, list):
                subvalue = [subvalue]
            for subsubvalue in subvalue:
                write_cb('<' + escape.utf8(name) + '>')
                _render_parts(subsubvalue, write_cb)
                write_cb('</' + escape.utf8(name) + '>')
    else:
        raise Exception("Unknown S3 value type %r", value)

def get_argument(request, key, default_value):
    """Returns the request's value at key, or default_value
    if not found
    """
    if key in request.args:
        return request.args[key][0]
    return default_value

def get_context(request):
    """Returns the supplied request's context object"""
    try:
        # Authorization Header format: 'AWS <access>:<secret>'
        authorization_header = request.getHeader('Authorization')
        if not authorization_header:
            raise exception.NotAuthorized()
        auth_header_value = authorization_header.split(' ')[1]
        access, _ignored, secret = auth_header_value.rpartition(':')
        am = manager.AuthManager()
        (user, project) = am.authenticate(access,
                                          secret,
                                          {},
                                          request.method,
                                          request.getRequestHostname(),
                                          request.uri,
                                          headers=request.getAllHeaders(),
                                          check_type='s3')
        return context.RequestContext(user, project)
    except exception.Error as ex:
        logging.debug("Authentication Failure: %s", ex)
        raise exception.NotAuthorized()

class ErrorHandlingResource(resource.Resource):
    """Maps exceptions to 404 / 401 codes.  Won't work for
    exceptions thrown after NOT_DONE_YET is returned.
    """
    # TODO(unassigned) (calling-all-twisted-experts): This needs to be
    #                   plugged in to the right place in twisted...
    #                   This doesn't look like it's the right place
    #                   (consider exceptions in getChild; or after
    #                   NOT_DONE_YET is returned
    def render(self, request):
        """Renders the response as XML"""
        try:
            return resource.Resource.render(self, request)
        except exception.NotFound:
            request.setResponseCode(404)
            return ''
        except exception.NotAuthorized:
            request.setResponseCode(403)
            return ''

class S3(ErrorHandlingResource):
    """Implementation of an S3-like storage server based on local files."""
    def __init__(self):
        ErrorHandlingResource.__init__(self)

    def getChild(self, name, request): # pylint: disable-msg=C0103
        """Returns either the image or bucket resource"""
        request.context = get_context(request)
        if name == '':
            return self
        elif name == '_images':
            return ImagesResource()
        else:
            return BucketResource(name)

    def render_GET(self, request): # pylint: disable-msg=R0201
        """Renders the GET request for a list of buckets as XML"""
        logging.debug('List of buckets requested')
        buckets = [b for b in bucket.Bucket.all() \
                   if b.is_authorized(request.context)]

        render_xml(request, {"ListAllMyBucketsResult": {
            "Buckets": {"Bucket": [b.metadata for b in buckets]},
        }})
        return server.NOT_DONE_YET

class BucketResource(ErrorHandlingResource):
    """A web resource containing an S3-like bucket"""
    def __init__(self, name):
        resource.Resource.__init__(self)
        self.name = name

    def getChild(self, name, request):
        """Returns the bucket resource itself, or the object resource
        the bucket contains if a name is supplied
        """
        if name == '':
            return self
        else:
            return ObjectResource(bucket.Bucket(self.name), name)

    def render_GET(self, request):
        "Returns the keys for the bucket resource"""
        logging.debug("List keys for bucket %s", self.name)

        try:
            bucket_object = bucket.Bucket(self.name)
        except exception.NotFound:
            return error.NoResource(message="No such bucket").render(request)

        if not bucket_object.is_authorized(request.context):
            raise exception.NotAuthorized()

        prefix = get_argument(request, "prefix", u"")
        marker = get_argument(request, "marker", u"")
        max_keys = int(get_argument(request, "max-keys", 1000))
        terse = int(get_argument(request, "terse", 0))

        results = bucket_object.list_keys(prefix=prefix,
                                          marker=marker,
                                          max_keys=max_keys,
                                          terse=terse)
        render_xml(request, {"ListBucketResult": results})
        return server.NOT_DONE_YET

    def render_PUT(self, request):
        "Creates the bucket resource"""
        logging.debug("Creating bucket %s", self.name)
        logging.debug("calling bucket.Bucket.create(%r, %r)",
                      self.name,
                      request.context)
        bucket.Bucket.create(self.name, request.context)
        request.finish()
        return server.NOT_DONE_YET

    def render_DELETE(self, request):
        """Deletes the bucket resource"""
        logging.debug("Deleting bucket %s", self.name)
        bucket_object = bucket.Bucket(self.name)

        if not bucket_object.is_authorized(request.context):
            raise exception.NotAuthorized()

        bucket_object.delete()
        request.setResponseCode(204)
        return ''

class ObjectResource(ErrorHandlingResource):
    """The resource returned from a bucket"""
    def __init__(self, bucket, name):
        resource.Resource.__init__(self)
        self.bucket = bucket
        self.name = name

    def render_GET(self, request):
        """Returns the object

        Raises NotAuthorized if user in request context is not
        authorized to delete the object.
        """
        logging.debug("Getting object: %s / %s", self.bucket.name, self.name)

        if not self.bucket.is_authorized(request.context):
            raise exception.NotAuthorized()

        obj = self.bucket[urllib.unquote(self.name)]
        request.setHeader("Content-Type", "application/unknown")
        request.setHeader("Last-Modified",
                          datetime.datetime.utcfromtimestamp(obj.mtime))
        request.setHeader("Etag", '"' + obj.md5 + '"')
        return static.File(obj.path).render_GET(request)

    def render_PUT(self, request):
        """Modifies/inserts the object and returns a result code

        Raises NotAuthorized if user in request context is not
        authorized to delete the object.
        """
        logging.debug("Putting object: %s / %s", self.bucket.name, self.name)

        if not self.bucket.is_authorized(request.context):
            raise exception.NotAuthorized()

        key = urllib.unquote(self.name)
        request.content.seek(0, 0)
        self.bucket[key] = request.content.read()
        request.setHeader("Etag", '"' + self.bucket[key].md5 + '"')
        finish(request)
        return server.NOT_DONE_YET

    def render_DELETE(self, request):
        """Deletes the object and returns a result code

        Raises NotAuthorized if user in request context is not
        authorized to delete the object.
        """

        logging.debug("Deleting object: %s / %s",
                      self.bucket.name,
                      self.name)

        if not self.bucket.is_authorized(request.context):
            raise exception.NotAuthorized()

        del self.bucket[urllib.unquote(self.name)]
        request.setResponseCode(204)
        return ''

class ImageResource(ErrorHandlingResource):
    """A web resource representing a single image"""
    isLeaf = True

    def __init__(self, name):
        resource.Resource.__init__(self)
        self.img = image.Image(name)

    def render_GET(self, request):
        """Returns the image file"""
        if not self.img.is_authorized(request.context, True):
            raise exception.NotAuthorized()
        return static.File(self.img.image_path,
                           defaultType='application/octet-stream'
                          ).render_GET(request)

class ImagesResource(resource.Resource):
    """A web resource representing a list of images"""
    def getChild(self, name, _request):
        """Returns itself or an ImageResource if no name given"""
        if name == '':
            return self
        else:
            return ImageResource(name)

    def render_GET(self, request): # pylint: disable-msg=R0201
        """ returns a json listing of all images
            that a user has permissions to see """

        images = [i for i in image.Image.all() \
                  if i.is_authorized(request.context, readonly=True)]

        # Bug #617776:
        # We used to have 'type' in the image metadata, but this field
        # should be called 'imageType', as per the EC2 specification.
        # For compat with old metadata files we copy type to imageType if
        # imageType is not present.
        # For compat with euca2ools (and any other clients using the
        # incorrect name) we copy imageType to type.
        # imageType is primary if we end up with both in the metadata file
        # (which should never happen).
        def decorate(m):
            if 'imageType' not in m and 'type' in m:
                m[u'imageType'] = m['type']
            elif 'imageType' in m:
                m[u'type'] = m['imageType']
            if 'displayName' not in m:
                m[u'displayName'] = u''
            return m

        request.write(json.dumps([decorate(i.metadata) for i in images]))
        request.finish()
        return server.NOT_DONE_YET

    def render_PUT(self, request): # pylint: disable-msg=R0201
        """ create a new registered image """

        image_id = get_argument(request, 'image_id', u'')
        image_location = get_argument(request, 'image_location', u'')

        image_path = os.path.join(FLAGS.images_path, image_id)
        if not image_path.startswith(FLAGS.images_path) or \
           os.path.exists(image_path):
            raise exception.NotAuthorized()

        bucket_object = bucket.Bucket(image_location.split("/")[0])

        if not bucket_object.is_authorized(request.context):
            raise exception.NotAuthorized()

        p = multiprocessing.Process(target=image.Image.register_aws_image,
                args=(image_id, image_location, request.context))
        p.start()
        return ''

    def render_POST(self, request): # pylint: disable-msg=R0201
        """Update image attributes: public/private"""

        # image_id required for all requests
        image_id = get_argument(request, 'image_id', u'')
        image_object = image.Image(image_id)
        if not image_object.is_authorized(request.context):
            logging.debug("not authorized for render_POST in images")
            raise exception.NotAuthorized()

        operation = get_argument(request, 'operation', u'')
        if operation:
            # operation implies publicity toggle
            logging.debug("handling publicity toggle")
            image_object.set_public(operation=='add')
        else:
            # other attributes imply update
            logging.debug("update user fields")
            clean_args = {}
            for arg in request.args.keys():
                clean_args[arg] = request.args[arg][0]
            image_object.update_user_editable_fields(clean_args)
        return ''

    def render_DELETE(self, request): # pylint: disable-msg=R0201
        """Delete a registered image"""
        image_id = get_argument(request, "image_id", u"")
        image_object = image.Image(image_id)

        if not image_object.is_authorized(request.context):
            raise exception.NotAuthorized()

        image_object.delete()

        request.setResponseCode(204)
        return ''

def get_site():
    """Support for WSGI-like interfaces"""
    root = S3()
    site = server.Site(root)
    return site

def get_application():
    """Support WSGI-like interfaces"""
    factory = get_site()
    application = service.Application("objectstore")
    # Disabled because of lack of proper introspection in Twisted
    # or possibly different versions of twisted?
    # pylint: disable-msg=E1101
    objectStoreService = internet.TCPServer(FLAGS.s3_port, factory)
    objectStoreService.setServiceParent(application)
    return application
<|endoftext|>


<commit_before>import hoomd
from hoomd import _hoomd
from hoomd.filter import ParticleFilter, All
from hoomd.data.parameterdicts import ParameterDict
from hoomd.operation import Writer

class DCD(Writer):
    """Writes simulation snapshots in the DCD format.

    Args:
        filename (str): File name to write.
        trigger (hoomd.trigger.Trigger): Select the timesteps to write.
        filter (hoomd.filter.ParticleFilter): Select the particles to write.
            Defaults to `hoomd.filter.All`.
        overwrite (bool): When False, (the default) an existing DCD file will be appended to. When True, an existing DCD
                          file *filename* will be overwritten.
        unwrap_full (bool): When False, (the default) particle coordinates are always written inside the simulation box.
                            When True, particles will be unwrapped into their current box image before writing to the dcd file.
        unwrap_rigid (bool): When False, (the default) individual particles are written inside the simulation box which
               breaks up rigid bodies near box boundaries. When True, particles belonging to the same rigid body will be
               unwrapped so that the body is continuous. The center of mass of the body remains in the simulation box, but
               some particles may be written just outside it. *unwrap_rigid* is ignored when *unwrap_full* is True.
        angle_z (bool): When True, the particle orientation angle is written to the z component (only useful for 2D simulations)

    On each timestep where `DCD` triggers, it writes the simulation snapshot the
    specified file in the DCD file format. DCD only stores particle positions,
    in distance units.

    Examples::

        hoomd.write.DCD("trajectory.dcd", hoomd.trigger.Periodic(1000))
        dcd = hoomd.write.DCD(filename="data/dump.dcd", trigger=hoomd.trigger.Periodic(100, 10))

    Warning:
        When you use dump.dcd to append to an existing dcd file:

        * The period must be the same or the time data in the file will not be consistent.
        * dump.dcd will not write out data at time steps that already are present in the dcd file to maintain a
          consistent timeline

    Attributes:
        filename (str): File name to write.
        trigger (hoomd.trigger.Trigger): Select the timesteps to write.
        filter (hoomd.filter.ParticleFilter): Select the particles to write.
        overwrite (bool): When False, an existing DCD file will be appended to. When True, an existing DCD
                          file *filename* will be overwritten.
        unwrap_full (bool): When False, particle coordinates are always written inside the simulation box.
                            When True, particles will be unwrapped into their current box image before writing to the dcd file.
        unwrap_rigid (bool): When False, individual particles are written inside the simulation box which
               breaks up rigid bodies near box boundaries. When True, particles belonging to the same rigid body will be
               unwrapped so that the body is continuous. The center of mass of the body remains in the simulation box, but
               some particles may be written just outside it. *unwrap_rigid* is ignored when *unwrap_full* is True.
        angle_z (bool): When True, the particle orientation angle is written to the z component
    """
    def __init__(self,
                 filename,
                 trigger,
                 filter=All(),
                 overwrite=False,
                 unwrap_full=False,
                 unwrap_rigid=False,
                 angle_z=False):

        # initialize base class
        super().__init__(trigger)
        self._param_dict.update(
            ParameterDict(filename=str(filename),
                          filter=ParticleFilter,
                          overwrite=bool(overwrite),
                          write_settings=[bool],
                          _defaults=dict(filter=filter,
                                         write_settings=[unwrap_full,
                                                         unwrap_rigid,
                                                         angle_z])))

    def _attach(self):
        write_settings = []
        if self.write_settings is not None:
            write_settings = [setting for setting in self.write_settings]
        group = self._simulation.state._get_group(self.filter)

        self._cpp_obj = _hoomd.DCDDumpWriter(self._simulation.state._cpp_sys_def,
                                             self.filename,
                                             int(self.trigger.period),
                                             group,
                                             self.overwrite)
        self._cpp_obj.setUnwrapFull(write_settings[0])
        self._cpp_obj.setUnwrapRigid(write_settings[1])
        self._cpp_obj.setAngleZ(write_settings[2])
        super()._attach()

    def enable(self):
        """ The DCD dump writer cannot be re-enabled """
        if self.enabled is False:
            raise RuntimeError('DCD output cannot be re-enabled after it has been disabled');

    def set_period(self, period):
        raise RuntimeError('The period of a DCD writer cannot be changed')
<commit_msg>Remove unnecessary methods<commit_after>import hoomd
from hoomd import _hoomd
from hoomd.filter import ParticleFilter, All
from hoomd.data.parameterdicts import ParameterDict
from hoomd.operation import Writer

class DCD(Writer):
    """Writes simulation snapshots in the DCD format.

    Args:
        filename (str): File name to write.
        trigger (hoomd.trigger.Trigger): Select the timesteps to write.
        filter (hoomd.filter.ParticleFilter): Select the particles to write.
            Defaults to `hoomd.filter.All`.
        overwrite (bool): When False, (the default) an existing DCD file will be appended to. When True, an existing DCD
                          file *filename* will be overwritten.
        unwrap_full (bool): When False, (the default) particle coordinates are always written inside the simulation box.
                            When True, particles will be unwrapped into their current box image before writing to the dcd file.
        unwrap_rigid (bool): When False, (the default) individual particles are written inside the simulation box which
               breaks up rigid bodies near box boundaries. When True, particles belonging to the same rigid body will be
               unwrapped so that the body is continuous. The center of mass of the body remains in the simulation box, but
               some particles may be written just outside it. *unwrap_rigid* is ignored when *unwrap_full* is True.
        angle_z (bool): When True, the particle orientation angle is written to the z component (only useful for 2D simulations)

    On each timestep where `DCD` triggers, it writes the simulation snapshot the
    specified file in the DCD file format. DCD only stores particle positions,
    in distance units.

    Examples::

        hoomd.write.DCD("trajectory.dcd", hoomd.trigger.Periodic(1000))
        dcd = hoomd.write.DCD(filename="data/dump.dcd", trigger=hoomd.trigger.Periodic(100, 10))

    Warning:
        When you use dump.dcd to append to an existing dcd file:

        * The period must be the same or the time data in the file will not be consistent.
        * dump.dcd will not write out data at time steps that already are present in the dcd file to maintain a
          consistent timeline

    Attributes:
        filename (str): File name to write.
        trigger (hoomd.trigger.Trigger): Select the timesteps to write.
        filter (hoomd.filter.ParticleFilter): Select the particles to write.
        overwrite (bool): When False, an existing DCD file will be appended to. When True, an existing DCD
                          file *filename* will be overwritten.
        unwrap_full (bool): When False, particle coordinates are always written inside the simulation box.
                            When True, particles will be unwrapped into their current box image before writing to the dcd file.
        unwrap_rigid (bool): When False, individual particles are written inside the simulation box which
               breaks up rigid bodies near box boundaries. When True, particles belonging to the same rigid body will be
               unwrapped so that the body is continuous. The center of mass of the body remains in the simulation box, but
               some particles may be written just outside it. *unwrap_rigid* is ignored when *unwrap_full* is True.
        angle_z (bool): When True, the particle orientation angle is written to the z component
    """
    def __init__(self,
                 filename,
                 trigger,
                 filter=All(),
                 overwrite=False,
                 unwrap_full=False,
                 unwrap_rigid=False,
                 angle_z=False):

        # initialize base class
        super().__init__(trigger)
        self._param_dict.update(
            ParameterDict(filename=str(filename),
                          filter=ParticleFilter,
                          overwrite=bool(overwrite),
                          write_settings=[bool],
                          _defaults=dict(filter=filter,
                                         write_settings=[unwrap_full,
                                                         unwrap_rigid,
                                                         angle_z])))

    def _attach(self):
        write_settings = []
        if self.write_settings is not None:
            write_settings = [setting for setting in self.write_settings]
        group = self._simulation.state._get_group(self.filter)

        self._cpp_obj = _hoomd.DCDDumpWriter(self._simulation.state._cpp_sys_def,
                                             self.filename,
                                             int(self.trigger.period),
                                             group,
                                             self.overwrite)
        self._cpp_obj.setUnwrapFull(write_settings[0])
        self._cpp_obj.setUnwrapRigid(write_settings[1])
        self._cpp_obj.setAngleZ(write_settings[2])
        super()._attach()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

# Copyright 2011-2018, Nigel Small
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from io import StringIO
from unittest import TestCase

from py2neo.data import DataList

class DataListTestCase(TestCase):

    def test_simple_usage(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        self.assertEqual(table.keys(), ["name", "age"])
        name_field = table.field(0)
        self.assertEqual(name_field["type"], str)
        self.assertEqual(name_field["optional"], False)
        age_field = table.field(1)
        self.assertEqual(age_field["type"], int)
        self.assertEqual(age_field["optional"], False)

    def test_missing_keys(self):
        with self.assertRaises(ValueError):
            _ = DataList([
                ["Alice", 33],
                ["Bob", 44],
                ["Carol", 55],
                ["Dave", 66],
            ])

    def test_optional_fields(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", None],
            [None, 66],
        ], keys=["name", "age"])
        self.assertEqual(table.keys(), ["name", "age"])
        name_field = table.field(0)
        self.assertEqual(name_field["type"], str)
        self.assertEqual(name_field["optional"], True)
        age_field = table.field(1)
        self.assertEqual(age_field["type"], int)
        self.assertEqual(age_field["optional"], True)

    def test_mixed_types(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55.5],
            ["Dave", 66.6],
        ], keys=["name", "age"])
        self.assertEqual(table.keys(), ["name", "age"])
        name_field = table.field(0)
        self.assertEqual(name_field["type"], str)
        self.assertEqual(name_field["optional"], False)
        age_field = table.field(1)
        self.assertEqual(set(age_field["type"]), {int, float})
        self.assertEqual(age_field["optional"], False)

    def test_fields_by_name_usage(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        self.assertEqual(table.keys(), ["name", "age"])
        name_field = table.field("name")
        self.assertEqual(name_field["type"], str)
        self.assertEqual(name_field["optional"], False)
        age_field = table.field("age")
        self.assertEqual(age_field["type"], int)
        self.assertEqual(age_field["optional"], False)
        with self.assertRaises(KeyError):
            _ = table.field("gender")

    def test_bad_typed_field_selector(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        with self.assertRaises(TypeError):
            _ = table.field(object)

    def test_repr(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = repr(table)
        self.assertEqual(out, ' name  | age \r\n'
                              '-------------\r\n'
                              ' Alice |  33 \r\n'
                              ' Bob   |  44 \r\n'
                              ' Carol |  55 \r\n'
                              ' Dave  |  66 \r\n')

    def test_write(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out)
        self.assertEqual(out.getvalue(), ' Alice | 33 \r\n'
                                         ' Bob   | 44 \r\n'
                                         ' Carol | 55 \r\n'
                                         ' Dave  | 66 \r\n')

    def test_write_with_newline_in_value(self):
        table = DataList([
            ["Alice\nSmith", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out)
        self.assertEqual(out.getvalue(), ' Alice | 33 \r\n'
                                         ' Smith |    \r\n'
                                         ' Bob   | 44 \r\n'
                                         ' Carol | 55 \r\n'
                                         ' Dave  | 66 \r\n')

    def test_write_with_style(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, header={"fg": "red"})
        self.assertEqual(out.getvalue(), ' name  | age \r\n'
                                         '-------------\r\n'
                                         ' Alice |  33 \r\n'
                                         ' Bob   |  44 \r\n'
                                         ' Carol |  55 \r\n'
                                         ' Dave  |  66 \r\n')

    def test_write_with_skip(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, skip=2)
        self.assertEqual(out.getvalue(), ' Carol | 55 \r\n'
                                         ' Dave  | 66 \r\n')

    def test_write_with_limit(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, limit=2)
        self.assertEqual(out.getvalue(), ' Alice | 33 \r\n'
                                         ' Bob   | 44 \r\n')

    def test_write_with_skip_and_limit(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, skip=1, limit=2)
        self.assertEqual(out.getvalue(), ' Bob   | 44 \r\n'
                                         ' Carol | 55 \r\n')

    def test_write_with_skip_and_limit_overflow(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, skip=1, limit=10)
        self.assertEqual(out.getvalue(), ' Bob   | 44 \r\n'
                                         ' Carol | 55 \r\n'
                                         ' Dave  | 66 \r\n')

    def test_write_csv(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out)
        self.assertEqual(out.getvalue(), 'Alice,33\r\n'
                                         'Bob,44\r\n'
                                         'Carol,55\r\n'
                                         'Dave,66\r\n')

    def test_write_csv_with_header(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out, header=True)
        self.assertEqual(out.getvalue(), 'name,age\r\n'
                                         'Alice,33\r\n'
                                         'Bob,44\r\n'
                                         'Carol,55\r\n'
                                         'Dave,66\r\n')

    def test_write_csv_with_header_style(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out, header={"fg": "cyan"})
        self.assertEqual(out.getvalue(), 'name,age\r\n'
                                         'Alice,33\r\n'
                                         'Bob,44\r\n'
                                         'Carol,55\r\n'
                                         'Dave,66\r\n')

    def test_write_csv_with_limit(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out, limit=2)
        self.assertEqual(out.getvalue(), 'Alice,33\r\n'
                                         'Bob,44\r\n')

    def test_write_csv_with_comma_in_value(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Smith, Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out)
        self.assertEqual(out.getvalue(), 'Alice,33\r\n'
                                         'Bob,44\r\n'
                                         'Carol,55\r\n'
                                         '"Smith, Dave",66\r\n')

    def test_write_csv_with_quotes_in_value(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave \"Nordberg\" Smith", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out)
        self.assertEqual(out.getvalue(), 'Alice,33\r\n'
                                         'Bob,44\r\n'
                                         'Carol,55\r\n'
                                         '"Dave ""Nordberg"" Smith",66\r\n')

    def test_write_csv_with_none_in_value(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", None],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out)
        self.assertEqual(out.getvalue(), 'Alice,33\r\n'
                                         'Bob,44\r\n'
                                         'Carol,55\r\n'
                                         'Dave,\r\n')

    def test_write_tsv(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_tsv(out)
        self.assertEqual(out.getvalue(), 'Alice\t33\r\n'
                                         'Bob\t44\r\n'
                                         'Carol\t55\r\n'
                                         'Dave\t66\r\n')
<commit_msg>DataList write in unicode<commit_after>#!/usr/bin/env python
# -*- encoding: utf-8 -*-

# Copyright 2011-2018, Nigel Small
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from io import StringIO
from unittest import TestCase

from py2neo.data import DataList

class DataListTestCase(TestCase):

    def test_simple_usage(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        self.assertEqual(table.keys(), ["name", "age"])
        name_field = table.field(0)
        self.assertEqual(name_field["type"], str)
        self.assertEqual(name_field["optional"], False)
        age_field = table.field(1)
        self.assertEqual(age_field["type"], int)
        self.assertEqual(age_field["optional"], False)

    def test_missing_keys(self):
        with self.assertRaises(ValueError):
            _ = DataList([
                ["Alice", 33],
                ["Bob", 44],
                ["Carol", 55],
                ["Dave", 66],
            ])

    def test_optional_fields(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", None],
            [None, 66],
        ], keys=["name", "age"])
        self.assertEqual(table.keys(), ["name", "age"])
        name_field = table.field(0)
        self.assertEqual(name_field["type"], str)
        self.assertEqual(name_field["optional"], True)
        age_field = table.field(1)
        self.assertEqual(age_field["type"], int)
        self.assertEqual(age_field["optional"], True)

    def test_mixed_types(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55.5],
            ["Dave", 66.6],
        ], keys=["name", "age"])
        self.assertEqual(table.keys(), ["name", "age"])
        name_field = table.field(0)
        self.assertEqual(name_field["type"], str)
        self.assertEqual(name_field["optional"], False)
        age_field = table.field(1)
        self.assertEqual(set(age_field["type"]), {int, float})
        self.assertEqual(age_field["optional"], False)

    def test_fields_by_name_usage(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        self.assertEqual(table.keys(), ["name", "age"])
        name_field = table.field("name")
        self.assertEqual(name_field["type"], str)
        self.assertEqual(name_field["optional"], False)
        age_field = table.field("age")
        self.assertEqual(age_field["type"], int)
        self.assertEqual(age_field["optional"], False)
        with self.assertRaises(KeyError):
            _ = table.field("gender")

    def test_bad_typed_field_selector(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        with self.assertRaises(TypeError):
            _ = table.field(object)

    def test_repr(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = repr(table)
        self.assertEqual(out, u' name  | age \r\n'
                              u'-------------\r\n'
                              u' Alice |  33 \r\n'
                              u' Bob   |  44 \r\n'
                              u' Carol |  55 \r\n'
                              u' Dave  |  66 \r\n')

    def test_write(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out)
        self.assertEqual(out.getvalue(), u' Alice | 33 \r\n'
                                         u' Bob   | 44 \r\n'
                                         u' Carol | 55 \r\n'
                                         u' Dave  | 66 \r\n')

    def test_write_with_newline_in_value(self):
        table = DataList([
            ["Alice\nSmith", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out)
        self.assertEqual(out.getvalue(), u' Alice | 33 \r\n'
                                         u' Smith |    \r\n'
                                         u' Bob   | 44 \r\n'
                                         u' Carol | 55 \r\n'
                                         u' Dave  | 66 \r\n')

    def test_write_with_style(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, header={"fg": "red"})
        self.assertEqual(out.getvalue(), u' name  | age \r\n'
                                         u'-------------\r\n'
                                         u' Alice |  33 \r\n'
                                         u' Bob   |  44 \r\n'
                                         u' Carol |  55 \r\n'
                                         u' Dave  |  66 \r\n')

    def test_write_with_skip(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, skip=2)
        self.assertEqual(out.getvalue(), u' Carol | 55 \r\n'
                                         u' Dave  | 66 \r\n')

    def test_write_with_limit(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, limit=2)
        self.assertEqual(out.getvalue(), u' Alice | 33 \r\n'
                                         u' Bob   | 44 \r\n')

    def test_write_with_skip_and_limit(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, skip=1, limit=2)
        self.assertEqual(out.getvalue(), u' Bob   | 44 \r\n'
                                         u' Carol | 55 \r\n')

    def test_write_with_skip_and_limit_overflow(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write(out, skip=1, limit=10)
        self.assertEqual(out.getvalue(), u' Bob   | 44 \r\n'
                                         u' Carol | 55 \r\n'
                                         u' Dave  | 66 \r\n')

    def test_write_csv(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out)
        self.assertEqual(out.getvalue(), u'Alice,33\r\n'
                                         u'Bob,44\r\n'
                                         u'Carol,55\r\n'
                                         u'Dave,66\r\n')

    def test_write_csv_with_header(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out, header=True)
        self.assertEqual(out.getvalue(), u'name,age\r\n'
                                         u'Alice,33\r\n'
                                         u'Bob,44\r\n'
                                         u'Carol,55\r\n'
                                         u'Dave,66\r\n')

    def test_write_csv_with_header_style(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out, header={"fg": "cyan"})
        self.assertEqual(out.getvalue(), u'name,age\r\n'
                                         u'Alice,33\r\n'
                                         u'Bob,44\r\n'
                                         u'Carol,55\r\n'
                                         u'Dave,66\r\n')

    def test_write_csv_with_limit(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out, limit=2)
        self.assertEqual(out.getvalue(), u'Alice,33\r\n'
                                         u'Bob,44\r\n')

    def test_write_csv_with_comma_in_value(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Smith, Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out)
        self.assertEqual(out.getvalue(), u'Alice,33\r\n'
                                         u'Bob,44\r\n'
                                         u'Carol,55\r\n'
                                         u'"Smith, Dave",66\r\n')

    def test_write_csv_with_quotes_in_value(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave \"Nordberg\" Smith", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out)
        self.assertEqual(out.getvalue(), u'Alice,33\r\n'
                                         u'Bob,44\r\n'
                                         u'Carol,55\r\n'
                                         u'"Dave ""Nordberg"" Smith",66\r\n')

    def test_write_csv_with_none_in_value(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", None],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_csv(out)
        self.assertEqual(out.getvalue(), u'Alice,33\r\n'
                                         u'Bob,44\r\n'
                                         u'Carol,55\r\n'
                                         u'Dave,\r\n')

    def test_write_tsv(self):
        table = DataList([
            ["Alice", 33],
            ["Bob", 44],
            ["Carol", 55],
            ["Dave", 66],
        ], keys=["name", "age"])
        out = StringIO()
        table.write_tsv(out)
        self.assertEqual(out.getvalue(), u'Alice\t33\r\n'
                                         u'Bob\t44\r\n'
                                         u'Carol\t55\r\n'
                                         u'Dave\t66\r\n')
<|endoftext|>