<commit_before>#!/usr/bin/python

# encodes to ogg

import os,sys,subprocess
import xml.etree.ElementTree

from process import process

from main.models import Client, Show, Location, Episode, Raw_File, Cut_List

mlt="""
<mlt>

  <producer id="title" resource="title.png" in="0" out="149" />
  <producer id="producer0" resource="/home/juser/vid/t2.dv" />

  <playlist id="playlist0">
    <entry id="clip" producer="producer0" in="500" out="690" />
  </playlist>

  <playlist id="playlist1">
    <entry producer="title"/>
  </playlist>

  <tractor id="tractor0">
       <multitrack>
         <track producer="playlist0"/>
         <track producer="playlist1"/>
       </multitrack>
       <transition
         mlt_service="luma" in="100" out="149" a_track="1" b_track="0"/>
   </tractor>

</mlt>
"""

def time2s(time):
    """ given 's.s' or 'h:m:s.s' returns s.s """
    if ':' in time:
        sec = reduce(lambda x, i: x*60 + i,
            map(float, time.split(':')))
    else:
        # print time, len(time), [c for c in time]
        sec = float(time)
    return sec

def time2f(time,fps):
    if time[-1]=='f':
        return int(time[:-1])
    else:
        return time2s(time)*fps

def time2b(time,fps,bpf,default):
    """
    converts the time stored in the db
      (as blank, seconds,  h:m:s.s )
    to the byte offset in the file.
    blank returns default, typically either 0 or filesize for start/end.
    fps is 25.0 for pal and 29.9 ntsc.
    bpf (bytes per frame) is 120000 for both.
    """
    if time:
        bytes = int(time2s(time)*fps)*bpf
    else:
        bytes = default
    return bytes

class enc(process):

  ready_state = 2

  def run_cmd(self,cmd):
    """
    given command list,
    if verbose prints stuff,
    runs it, returns pass/fail.
    """
    if self.options.verbose:
        print cmd
    print ' '.join(cmd)
    p=subprocess.Popen(cmd)
    p.wait()
    retcode=p.returncode
    if retcode:
        if self.options.verbose:
            print "transcode failed"
            print retcode
        ret = False
    else:
        ret = True
    return ret

  def mktitle(self, source, output_base, episode):
    """
    Make a title slide by filling in a pre-made svg with name/authors.
    melt uses librsvg which doesn't support flow,
    wich is needed for long titles, so render it to a .png using inkscape
    """
    text={
            'client': episode.location.show.client.name,
            'show': episode.location.show.name,
            'title': episode.name,
            'authors': episode.authors,
        }

    svg_in=open(source).read()
    tree=xml.etree.ElementTree.XMLID(svg_in)
    # print tree[1]
    # tree[1]['title'].text=name
    for key in ['client', 'show', 'title']:
        tree[1][key].text=text[key]

    prefix = "Featuring" if "," in text['authors'] else "By"
    tree[1]['presenternames'].text="%s %s" % (prefix,text['authors'])

    cooked_svg_name='%s.svg'%output_base
    open(cooked_svg_name,'w').write(xml.etree.ElementTree.tostring(tree[0]))
    png_name="%s.png"%output_base
    if self.options.verbose: print png_name
    cmd=["inkscape", cooked_svg_name, "--export-png", png_name]
    self.run_cmd(cmd)

    return png_name

  def mkmlt(self,title_img,episode,cls,rfs):
        """
        assemble a mlt playlist from:
        mlt template, title screen image,
        filter parameters (currently just audio)
        and cutlist+raw filenames
        """

# parse the xml into a tree of nodes
        tree= xml.etree.ElementTree.XMLID(mlt)

# set the title to the title slide we just made
        title=tree[1]['title']
        title.attrib['resource']=title_img

# get the dvfile placeholder and remove it from the tree
        dvfile=tree[1]['producer0']
        tree[0].remove(dvfile)

# add in the dv files
        pos = 1
        for rf in rfs:
            print rf
            dvfile.attrib['id']="producer%s"%rf.id
# hack to use .ogg instead of .dv
            # rawpathname = os.path.join(self.episode_dir,rf.basename()+".ogg")
# unhack to use .dv
            rawpathname = os.path.join(self.episode_dir,rf.filename)

            dvfile.attrib['resource']=rawpathname
            # new=xml.etree.ElementTree.SubElement(tree[0],'producer', dvfile.attrib )
            new=xml.etree.ElementTree.Element('producer', dvfile.attrib )
            tree[0].insert(pos,new)
            pos+=1

# get the clip placeholder, the playlist and remove the clip from the playlist
        clip=tree[1]['clip']
        playlist=tree[1]['playlist0']
        playlist.remove(clip)

# add in the clips
        pos = 0
        # for id,start,end in [(1,'500','690'),(2,'1000',None)]:
        for cl in cls:
            print cl
            clip.attrib['id']="clip%s"%cl.id
            clip.attrib['producer']="producer%s"%cl.raw_file.id
            clip.attrib['in']=str(time2f(cl.start,self.fps)) if cl.start else '0'
            clip.attrib['out']=str(time2f(cl.end,self.fps)) if cl.end else '999999'
            new=xml.etree.ElementTree.Element('entry', clip.attrib )
            playlist.insert(pos,new)
            pos+=1

# add volume tweeks
            """
    <filter mlt_service="channelcopy" from="1" to="0" />
    <filter mlt_service="volume" max_gain="30" normalise="28" />
            """
        if self.options.normalize:
            new=xml.etree.ElementTree.Element('filter',
                {'mlt_service':'volume',
                'max_gain':'1',
                'limiter':'1',
                'normalise':self.options.normalize} )
            playlist.insert(pos,new)

        if self.options.channelcopy:
            # channelcopy should be 01 or 10.
            fro,to=list(self.options.channelcopy)
            new=xml.etree.ElementTree.Element('filter',
                {'mlt_service':'channelcopy',
                'from':fro, 'to':to} )
            playlist.insert(pos,new)

        if self.options.verbose: xml.etree.ElementTree.dump(tree[0])

        mlt_pathname = os.path.join(self.show_dir, "tmp", "%s.mlt"%episode.slug)
        open(mlt_pathname,'w').write(xml.etree.ElementTree.tostring(tree[0]))

        return mlt_pathname

  def mk_title_dv(self, mlt_pathname, episode):

        # make an intro .dv from title and audio
        cmd="melt -verbose -profile dv_%s %s in=0 out=149 -consumer avformat:%s pix_fmt=yuv411p"
        dv_pathname = os.path.join(
            self.show_dir, "tmp", "%s_title.dv"%episode.slug)
        cmd = cmd % ( self.options.format.lower(), mlt_pathname, dv_pathname)
        ret = self.run_cmd(cmd.split())
# if not ret: raise something
        return dv_pathname

  def run_melt(self, mlt_pathname, episode):
        def one_format(ext, acodec, vcodec):
            if self.options.verbose:
                print "checking %s in %s" % (ext,self.options.output_format)
            if ext in self.options.output_format:
              cmd="melt -verbose -profile dv_%s %s -consumer avformat:%s acodec=%s ab=128k ar=44100 vcodec=%s minrate=0 b=900k progressive=1 deinterlace_method=onefield"
              out_pathname = os.path.join(
                self.show_dir, ext, "%s.%s"%(episode.slug,ext))
              cmd = cmd % ( self.options.format.lower(),
                mlt_pathname, out_pathname, acodec, vcodec)
              ret = self.run_cmd(cmd.split())
              if ret:
                  if not os.exists(out_pathname):
                      print "melt returned 0, but no output: %s" % out_pathname
                      ret=False

              return ret
            # this is the case where we don't do this format,
            # so don't flag as error
            return True

        ret = True
        ret = ret and one_format("ogg", "vorbis", "libtheora")
        ret = ret and one_format("flv", "libmp3lame", "flv")
        ret = ret and one_format("mp4", "libmp3lame", "mpeg4")

        return ret

  def mkdv(self, mlt_pathname, episode, cls ):
        """
        assemble parts into a master .dv file
        """
        title_dv=self.mk_title_dv(mlt_pathname, episode)

        # make a new dv file using just the frames to encode
        dvpathname = os.path.join(self.episode_dir,episode.slug+".dv")
        if self.options.verbose:
            print "making %s - may take awhile..." % dvpathname
        outf=open(dvpathname,'wb')

        # splice in the intro dv make by melt()
        title=open(title_dv,'rb').read()
        title_bytes=len(title)
        outf.write(title)

        for c in cls:
            if self.options.verbose:
                print (c.raw_file.filename, c.start,c.end)
            rawpathname = os.path.join(self.episode_dir,c.raw_file.filename)
            inf=open(rawpathname,'rb')
            inf.seek(time2b(c.start,self.fps,self.bpf,0)+title_bytes)
            # there is a problem if the first clip is shorter than the title.
# the next clip will start at 0,
# which is part of the title, so it will play twice.  oh well.
            title_bytes=0
            size=os.fstat(inf.fileno()).st_size
            end = time2b(c.end,self.fps,self.bpf,size)
            while inf.tell()<end:
                outf.write(inf.read(self.bpf))
            inf.close()
        outf.close()

        return dvpathname

  def dv2theora(self,episode,title_dv,cls,rfs):
        """
        transcode dv ot ogg
        Not much point in this now that the title side gets made
        because now there is never the case of one .dv file
        """
        oggpathname = os.path.join(self.show_dir, "ogv", "%s.ogv"%episode.slug)
        # cmd="ffmpeg2theora --videoquality 5 -V 600 --audioquality 5 --speedlevel 0 --optimize --keyint 256 --channels 1".split()
        cmd="ffmpeg2theora --videoquality 5 -V 600 --audioquality 5 --keyint 256 --channels 1".split()
        cmd+=['--output',oggpathname]

        if len(cls)==1 and not title_dv:
            # use the raw dv file and ffmpeg2theora params to trim
            c=cls[0]
            if c.start: cmd+=['--starttime',str(time2s(c.start))]
            if c.end: cmd+=['--endtime',str(time2s(c.end))]
            dvpathname = os.path.join(self.episode_dir,c.raw_file.filename)
        else:
            dvpathname = self.mkdv(episode,title_dv,cls,rfs)

        cmd+=[dvpathname]

        return cmd

  def process_ep(self,episode):
    # print episode
    ret = False
    cls = Cut_List.objects.filter(
        episode=episode, apply=True).order_by('sequence')
    # print len(cls), episode.name.__repr__()
    print episode.name
    for cl in cls:
        print cl.start, cl.end

    if cls:

# make a title slide
        template = os.path.join(self.show_dir, "bling", "title.svg")
        title_base = os.path.join(self.show_dir, "tmp", episode.slug)
        title_img=self.mktitle(template, title_base, episode)

# get list of raw footage for this episode
        rfs = Raw_File.objects.filter(
            cut_list__episode=episode).exclude(trash=True).distinct()

# make a .mlt file for this episode
        mlt = self.mkmlt(title_img,episode,cls,rfs)

# do the final encoding:
# using melt
        self.run_melt(mlt, episode)

# using script or ffmpeg2theora
        if self.options.enc_script or "ogv" in self.options.output_format:
          # consolidate all the dv files into one
          dvpathname = self.mkdv(mlt,episode,cls)
          if self.options.enc_script:
            cmd = [self.options.enc_script,
                    dvpathname, self.show_dir, episode.slug]
            ret = self.run_cmd(cmd)

          if "ogv" in self.options.output_format:
              oggpathname = os.path.join(
                self.show_dir, "ogv", "%s.ogv"%episode.slug)
              cmd="ffmpeg2theora --videoquality 5 -V 600 --audioquality 5" \
                " --speedlevel 0 --optimize --keyint 256" \
                " --channels 1".split()
              cmd+=[dvpathname,'--output',oggpathname]
              ret = self.run_cmd(cmd)

    else:
        print "No cutlist found."

    return ret

  def add_more_options(self, parser):
        parser.add_option('--output-format', default='',
          help='list of format(s) to output' )
        parser.add_option('--enc_script',
          help='encode shell script' )
        parser.add_option('--channelcopy',
          help='copy left to right (10) or right to left (01)' )
        parser.add_option('--normalize',
          help='normalise audio' )

if __name__ == '__main__':
    p=enc()
    p.main()

<commit_msg>exist is in os.path<commit_after>#!/usr/bin/python

# encodes to ogg

import os,sys,subprocess
import xml.etree.ElementTree

from process import process

from main.models import Client, Show, Location, Episode, Raw_File, Cut_List

mlt="""
<mlt>

  <producer id="title" resource="title.png" in="0" out="149" />
  <producer id="producer0" resource="/home/juser/vid/t2.dv" />

  <playlist id="playlist0">
    <entry id="clip" producer="producer0" in="500" out="690" />
  </playlist>

  <playlist id="playlist1">
    <entry producer="title"/>
  </playlist>

  <tractor id="tractor0">
       <multitrack>
         <track producer="playlist0"/>
         <track producer="playlist1"/>
       </multitrack>
       <transition
         mlt_service="luma" in="100" out="149" a_track="1" b_track="0"/>
   </tractor>

</mlt>
"""

def time2s(time):
    """ given 's.s' or 'h:m:s.s' returns s.s """
    if ':' in time:
        sec = reduce(lambda x, i: x*60 + i,
            map(float, time.split(':')))
    else:
        # print time, len(time), [c for c in time]
        sec = float(time)
    return sec

def time2f(time,fps):
    if time[-1]=='f':
        return int(time[:-1])
    else:
        return time2s(time)*fps

def time2b(time,fps,bpf,default):
    """
    converts the time stored in the db
      (as blank, seconds,  h:m:s.s )
    to the byte offset in the file.
    blank returns default, typically either 0 or filesize for start/end.
    fps is 25.0 for pal and 29.9 ntsc.
    bpf (bytes per frame) is 120000 for both.
    """
    if time:
        bytes = int(time2s(time)*fps)*bpf
    else:
        bytes = default
    return bytes

class enc(process):

  ready_state = 2

  def run_cmd(self,cmd):
    """
    given command list,
    if verbose prints stuff,
    runs it, returns pass/fail.
    """
    if self.options.verbose:
        print cmd
    print ' '.join(cmd)
    p=subprocess.Popen(cmd)
    p.wait()
    retcode=p.returncode
    if retcode:
        if self.options.verbose:
            print "transcode failed"
            print retcode
        ret = False
    else:
        ret = True
    return ret

  def mktitle(self, source, output_base, episode):
    """
    Make a title slide by filling in a pre-made svg with name/authors.
    melt uses librsvg which doesn't support flow,
    wich is needed for long titles, so render it to a .png using inkscape
    """
    text={
            'client': episode.location.show.client.name,
            'show': episode.location.show.name,
            'title': episode.name,
            'authors': episode.authors,
        }

    svg_in=open(source).read()
    tree=xml.etree.ElementTree.XMLID(svg_in)
    # print tree[1]
    # tree[1]['title'].text=name
    for key in ['client', 'show', 'title']:
        tree[1][key].text=text[key]

    prefix = "Featuring" if "," in text['authors'] else "By"
    tree[1]['presenternames'].text="%s %s" % (prefix,text['authors'])

    cooked_svg_name='%s.svg'%output_base
    open(cooked_svg_name,'w').write(xml.etree.ElementTree.tostring(tree[0]))
    png_name="%s.png"%output_base
    if self.options.verbose: print png_name
    cmd=["inkscape", cooked_svg_name, "--export-png", png_name]
    self.run_cmd(cmd)

    return png_name

  def mkmlt(self,title_img,episode,cls,rfs):
        """
        assemble a mlt playlist from:
        mlt template, title screen image,
        filter parameters (currently just audio)
        and cutlist+raw filenames
        """

# parse the xml into a tree of nodes
        tree= xml.etree.ElementTree.XMLID(mlt)

# set the title to the title slide we just made
        title=tree[1]['title']
        title.attrib['resource']=title_img

# get the dvfile placeholder and remove it from the tree
        dvfile=tree[1]['producer0']
        tree[0].remove(dvfile)

# add in the dv files
        pos = 1
        for rf in rfs:
            print rf
            dvfile.attrib['id']="producer%s"%rf.id
# hack to use .ogg instead of .dv
            # rawpathname = os.path.join(self.episode_dir,rf.basename()+".ogg")
# unhack to use .dv
            rawpathname = os.path.join(self.episode_dir,rf.filename)

            dvfile.attrib['resource']=rawpathname
            # new=xml.etree.ElementTree.SubElement(tree[0],'producer', dvfile.attrib )
            new=xml.etree.ElementTree.Element('producer', dvfile.attrib )
            tree[0].insert(pos,new)
            pos+=1

# get the clip placeholder, the playlist and remove the clip from the playlist
        clip=tree[1]['clip']
        playlist=tree[1]['playlist0']
        playlist.remove(clip)

# add in the clips
        pos = 0
        # for id,start,end in [(1,'500','690'),(2,'1000',None)]:
        for cl in cls:
            print cl
            clip.attrib['id']="clip%s"%cl.id
            clip.attrib['producer']="producer%s"%cl.raw_file.id
            clip.attrib['in']=str(time2f(cl.start,self.fps)) if cl.start else '0'
            clip.attrib['out']=str(time2f(cl.end,self.fps)) if cl.end else '999999'
            new=xml.etree.ElementTree.Element('entry', clip.attrib )
            playlist.insert(pos,new)
            pos+=1

# add volume tweeks
            """
    <filter mlt_service="channelcopy" from="1" to="0" />
    <filter mlt_service="volume" max_gain="30" normalise="28" />
            """
        if self.options.normalize:
            new=xml.etree.ElementTree.Element('filter',
                {'mlt_service':'volume',
                'max_gain':'1',
                'limiter':'1',
                'normalise':self.options.normalize} )
            playlist.insert(pos,new)

        if self.options.channelcopy:
            # channelcopy should be 01 or 10.
            fro,to=list(self.options.channelcopy)
            new=xml.etree.ElementTree.Element('filter',
                {'mlt_service':'channelcopy',
                'from':fro, 'to':to} )
            playlist.insert(pos,new)

        if self.options.verbose: xml.etree.ElementTree.dump(tree[0])

        mlt_pathname = os.path.join(self.show_dir, "tmp", "%s.mlt"%episode.slug)
        open(mlt_pathname,'w').write(xml.etree.ElementTree.tostring(tree[0]))

        return mlt_pathname

  def mk_title_dv(self, mlt_pathname, episode):

        # make an intro .dv from title and audio
        cmd="melt -verbose -profile dv_%s %s in=0 out=149 -consumer avformat:%s pix_fmt=yuv411p"
        dv_pathname = os.path.join(
            self.show_dir, "tmp", "%s_title.dv"%episode.slug)
        cmd = cmd % ( self.options.format.lower(), mlt_pathname, dv_pathname)
        ret = self.run_cmd(cmd.split())
# if not ret: raise something
        return dv_pathname

  def run_melt(self, mlt_pathname, episode):
        def one_format(ext, acodec, vcodec):
            if self.options.verbose:
                print "checking %s in %s" % (ext,self.options.output_format)
            if ext in self.options.output_format:
              cmd="melt -verbose -profile dv_%s %s -consumer avformat:%s acodec=%s ab=128k ar=44100 vcodec=%s minrate=0 b=900k progressive=1 deinterlace_method=onefield"
              out_pathname = os.path.join(
                self.show_dir, ext, "%s.%s"%(episode.slug,ext))
              cmd = cmd % ( self.options.format.lower(),
                mlt_pathname, out_pathname, acodec, vcodec)
              ret = self.run_cmd(cmd.split())
              if ret:
                  if not os.path.exists(out_pathname):
                      print "melt returned 0, but no output: %s" % out_pathname
                      ret=False

              return ret
            # this is the case where we don't do this format,
            # so don't flag as error
            return True

        ret = True
        ret = ret and one_format("ogg", "vorbis", "libtheora")
        ret = ret and one_format("flv", "libmp3lame", "flv")
        ret = ret and one_format("mp4", "libmp3lame", "mpeg4")

        return ret

  def mkdv(self, mlt_pathname, episode, cls ):
        """
        assemble parts into a master .dv file
        """
        title_dv=self.mk_title_dv(mlt_pathname, episode)

        # make a new dv file using just the frames to encode
        dvpathname = os.path.join(self.episode_dir,episode.slug+".dv")
        if self.options.verbose:
            print "making %s - may take awhile..." % dvpathname
        outf=open(dvpathname,'wb')

        # splice in the intro dv make by melt()
        title=open(title_dv,'rb').read()
        title_bytes=len(title)
        outf.write(title)

        for c in cls:
            if self.options.verbose:
                print (c.raw_file.filename, c.start,c.end)
            rawpathname = os.path.join(self.episode_dir,c.raw_file.filename)
            inf=open(rawpathname,'rb')
            inf.seek(time2b(c.start,self.fps,self.bpf,0)+title_bytes)
            # there is a problem if the first clip is shorter than the title.
# the next clip will start at 0,
# which is part of the title, so it will play twice.  oh well.
            title_bytes=0
            size=os.fstat(inf.fileno()).st_size
            end = time2b(c.end,self.fps,self.bpf,size)
            while inf.tell()<end:
                outf.write(inf.read(self.bpf))
            inf.close()
        outf.close()

        return dvpathname

  def dv2theora(self,episode,title_dv,cls,rfs):
        """
        transcode dv ot ogg
        Not much point in this now that the title side gets made
        because now there is never the case of one .dv file
        """
        oggpathname = os.path.join(self.show_dir, "ogv", "%s.ogv"%episode.slug)
        # cmd="ffmpeg2theora --videoquality 5 -V 600 --audioquality 5 --speedlevel 0 --optimize --keyint 256 --channels 1".split()
        cmd="ffmpeg2theora --videoquality 5 -V 600 --audioquality 5 --keyint 256 --channels 1".split()
        cmd+=['--output',oggpathname]

        if len(cls)==1 and not title_dv:
            # use the raw dv file and ffmpeg2theora params to trim
            c=cls[0]
            if c.start: cmd+=['--starttime',str(time2s(c.start))]
            if c.end: cmd+=['--endtime',str(time2s(c.end))]
            dvpathname = os.path.join(self.episode_dir,c.raw_file.filename)
        else:
            dvpathname = self.mkdv(episode,title_dv,cls,rfs)

        cmd+=[dvpathname]

        return cmd

  def process_ep(self,episode):
    # print episode
    ret = False
    cls = Cut_List.objects.filter(
        episode=episode, apply=True).order_by('sequence')
    # print len(cls), episode.name.__repr__()
    print episode.name
    for cl in cls:
        print cl.start, cl.end

    if cls:

# make a title slide
        template = os.path.join(self.show_dir, "bling", "title.svg")
        title_base = os.path.join(self.show_dir, "tmp", episode.slug)
        title_img=self.mktitle(template, title_base, episode)

# get list of raw footage for this episode
        rfs = Raw_File.objects.filter(
            cut_list__episode=episode).exclude(trash=True).distinct()

# make a .mlt file for this episode
        mlt = self.mkmlt(title_img,episode,cls,rfs)

# do the final encoding:
# using melt
        self.run_melt(mlt, episode)

# using script or ffmpeg2theora
        if self.options.enc_script or "ogv" in self.options.output_format:
          # consolidate all the dv files into one
          dvpathname = self.mkdv(mlt,episode,cls)
          if self.options.enc_script:
            cmd = [self.options.enc_script,
                    dvpathname, self.show_dir, episode.slug]
            ret = self.run_cmd(cmd)

          if "ogv" in self.options.output_format:
              oggpathname = os.path.join(
                self.show_dir, "ogv", "%s.ogv"%episode.slug)
              cmd="ffmpeg2theora --videoquality 5 -V 600 --audioquality 5" \
                " --speedlevel 0 --optimize --keyint 256" \
                " --channels 1".split()
              cmd+=[dvpathname,'--output',oggpathname]
              ret = self.run_cmd(cmd)

    else:
        print "No cutlist found."

    return ret

  def add_more_options(self, parser):
        parser.add_option('--output-format', default='',
          help='list of format(s) to output' )
        parser.add_option('--enc_script',
          help='encode shell script' )
        parser.add_option('--channelcopy',
          help='copy left to right (10) or right to left (01)' )
        parser.add_option('--normalize',
          help='normalise audio' )

if __name__ == '__main__':
    p=enc()
    p.main()

<|endoftext|>


<commit_before>"""Test list views

Includes success tests for creating resources by POST to the list view
Error tests are in test_errors.py
"""

from django.core.urlresolvers import reverse
from tests import models
from tests.utils import dump_json
import pytest

pytestmark = pytest.mark.django_db

def test_empty_list(client):
    results = {
        "posts": [],
    }

    response = client.get(reverse("post-list"))

    assert response.content == dump_json(results)

def test_single_item_list(client):
    models.Person.objects.create(name="test")

    results = {
        "people": [
            {
                "id": "1",
                "href": "http://testserver/people/1/",
                "name": "test",
            },
        ]
    }

    response = client.get(reverse("person-list"))

    assert response.content == dump_json(results)

def test_multiple_item_list(client):
    models.Person.objects.create(name="test")
    models.Person.objects.create(name="other")

    results = {
        "people": [
            {
                "id": "1",
                "href": "http://testserver/people/1/",
                "name": "test",
            },
            {
                "id": "2",
                "href": "http://testserver/people/2/",
                "name": "other",
            },
        ]
    }

    response = client.get(reverse("person-list"))

    assert response.content == dump_json(results)

def test_create_person_success(client):
    data = dump_json({
        "people": {
            "name": "Jason Api"
        }
    })
    results = {
        "people": {
            "id": "1",
            "href": "http://testserver/people/1/",
            "name": "Jason Api",
        }
    }

    response = client.post(
        reverse("person-list"), data=data,
        content_type="application/vnd.api+json")

    assert response.status_code == 201
    assert response.content == dump_json(results)
    assert response['content-type'] == 'application/vnd.api+json'
    person = models.Person.objects.get()
    assert person.name == 'Jason Api'

def test_create_post_success(client):
    author = models.Person.objects.create(name="The Author")

    # Should author be under links?
    # data = dump_json({
    #     "posts": {
    #         "title": "This is the title",
    #         "links": {
    #             "author": author.pk,
    #         },
    #     }
    # })
    # Current presentation
    data = dump_json({
        "posts": {
            "title": "This is the title",
            "author": "http://testserver/people/1/",
        }
    })

    response = client.post(
        reverse("post-list"), data=data,
        content_type="application/vnd.api+json")
    assert response.status_code == 201
    assert response['content-type'] == 'application/vnd.api+json'

    post = models.Post.objects.get()
    results = {
        "posts": {
            "id": str(post.pk),
            "href": "http://testserver/posts/%s/" % post.pk,
            "title": "This is the title",
            "links": {
                "author": str(author.pk),
                "comments": []
            }
        },
        "links": {
            "posts.author": {
                "href": "http://testserver/people/{posts.author}/",
                "type": "people"
            },
            "posts.comments": {
                "href": "http://testserver/comments/{posts.comments}/",
                "type": "comments"
            }
        },
    }
    assert response.content == dump_json(results)
<commit_msg>Switch creation test to use links<commit_after>"""Test list views

Includes success tests for creating resources by POST to the list view
Error tests are in test_errors.py
"""

from django.core.urlresolvers import reverse
from tests import models
from tests.utils import dump_json
import pytest

pytestmark = pytest.mark.django_db

def test_empty_list(client):
    results = {
        "posts": [],
    }

    response = client.get(reverse("post-list"))

    assert response.content == dump_json(results)

def test_single_item_list(client):
    models.Person.objects.create(name="test")

    results = {
        "people": [
            {
                "id": "1",
                "href": "http://testserver/people/1/",
                "name": "test",
            },
        ]
    }

    response = client.get(reverse("person-list"))

    assert response.content == dump_json(results)

def test_multiple_item_list(client):
    models.Person.objects.create(name="test")
    models.Person.objects.create(name="other")

    results = {
        "people": [
            {
                "id": "1",
                "href": "http://testserver/people/1/",
                "name": "test",
            },
            {
                "id": "2",
                "href": "http://testserver/people/2/",
                "name": "other",
            },
        ]
    }

    response = client.get(reverse("person-list"))

    assert response.content == dump_json(results)

def test_create_person_success(client):
    data = dump_json({
        "people": {
            "name": "Jason Api"
        }
    })
    results = {
        "people": {
            "id": "1",
            "href": "http://testserver/people/1/",
            "name": "Jason Api",
        }
    }

    response = client.post(
        reverse("person-list"), data=data,
        content_type="application/vnd.api+json")

    assert response.status_code == 201
    assert response.content == dump_json(results)
    assert response['content-type'] == 'application/vnd.api+json'
    person = models.Person.objects.get()
    assert person.name == 'Jason Api'

def test_create_post_success(client):
    author = models.Person.objects.create(name="The Author")

    data = dump_json({
        "posts": {
            "title": "This is the title",
            "links": {
                "author": author.pk,
            },
        }
    })

    response = client.post(
        reverse("post-list"), data=data,
        content_type="application/vnd.api+json")
    assert response.status_code == 201
    assert response['content-type'] == 'application/vnd.api+json'

    post = models.Post.objects.get()
    results = {
        "posts": {
            "id": str(post.pk),
            "href": "http://testserver/posts/%s/" % post.pk,
            "title": "This is the title",
            "links": {
                "author": str(author.pk),
                "comments": []
            }
        },
        "links": {
            "posts.author": {
                "href": "http://testserver/people/{posts.author}/",
                "type": "people"
            },
            "posts.comments": {
                "href": "http://testserver/comments/{posts.comments}/",
                "type": "comments"
            }
        },
    }
    assert response.content == dump_json(results)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contains the comment Model."""

__authors__ = [
  '"Daniel Hans" <daniel.m.hans@gmail.com>',
  '"Matthew Wilkes" <matthew@matthewwilkes.co.uk>',
]

from google.appengine.ext import db

from django.utils.translation import ugettext

import soc.models.base
import soc.models.role
import soc.models.user
import soc.models.linkable

class Comment(soc.models.linkable.Linkable):
  """Model of a comment on a work.

  A comment is usually associated with a Work or a Proposal,
  for example a Document or a Student Proposal, and with a user, the author.
  There are two types of comment, public (i.e. visible to the student),
  or private (i.e. visible to programme/club staff). Neither type are
  visible to people who are not connected to the work being commented on.
  """

  #: A required many:1 relationship with a comment entity indicating
  #: the user who provided that comment.
  author = db.ReferenceProperty(reference_class=soc.models.user.User,
                                required=True, collection_name="commented")

  #: The rich textual content of this comment
  content = db.TextProperty(verbose_name=ugettext('Content'))

  #: Indicated if the comment should be visible to the appropriate student
  is_public = db.BooleanProperty(verbose_name=ugettext('Public comment'))

  #: Date when the comment was added
  created = db.DateTimeProperty(auto_now_add=True)

  #: date when the work was last modified
  modified = db.DateTimeProperty(auto_now=True)

  # indicating wich user last modified the work. Used in displaying Work
  modified_by = db.ReferenceProperty(reference_class=soc.models.user.User,
                                     required=False,
                                     collection_name="modified_comments",
                                     verbose_name=ugettext('Modified by'))

class NewComment(soc.models.base.ModelWithFieldAttributes):
  """Model of a comment on a work.

  A comment is usually associated with a Work or a Proposal,
  for example a Document or a Student Proposal, and with a user, the author.
  There are two types of comment, public (i.e. visible to the student),
  or private (i.e. visible to programme/club staff). Neither type are
  visible to people who are not connected to the work being commented on.
  """

  #: A required many:1 relationship with a comment entity indicating
  #: the user who provided that comment.
  author = db.ReferenceProperty(reference_class=soc.models.role.Profile,
      required=True, collection_name="commented")

  #: The rich textual content of this comment
  content = db.TextProperty(verbose_name='')

  #: Indicated if the comment should be visible to the appropriate student
  is_private = db.BooleanProperty(verbose_name=ugettext('Private'))
  is_private.help_text = ugettext(
      'Whether this comment will only be seen by admins and mentors')

  #: Date when the comment was added
  created = db.DateTimeProperty(auto_now_add=True)
<commit_msg>is_private field is set to False by default.<commit_after>#!/usr/bin/env python2.5
#
# Copyright 2008 the Melange authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""This module contains the comment Model."""

__authors__ = [
  '"Daniel Hans" <daniel.m.hans@gmail.com>',
  '"Matthew Wilkes" <matthew@matthewwilkes.co.uk>',
]

from google.appengine.ext import db

from django.utils.translation import ugettext

import soc.models.base
import soc.models.role
import soc.models.user
import soc.models.linkable

class Comment(soc.models.linkable.Linkable):
  """Model of a comment on a work.

  A comment is usually associated with a Work or a Proposal,
  for example a Document or a Student Proposal, and with a user, the author.
  There are two types of comment, public (i.e. visible to the student),
  or private (i.e. visible to programme/club staff). Neither type are
  visible to people who are not connected to the work being commented on.
  """

  #: A required many:1 relationship with a comment entity indicating
  #: the user who provided that comment.
  author = db.ReferenceProperty(reference_class=soc.models.user.User,
                                required=True, collection_name="commented")

  #: The rich textual content of this comment
  content = db.TextProperty(verbose_name=ugettext('Content'))

  #: Indicated if the comment should be visible to the appropriate student
  is_public = db.BooleanProperty(verbose_name=ugettext('Public comment'))

  #: Date when the comment was added
  created = db.DateTimeProperty(auto_now_add=True)

  #: date when the work was last modified
  modified = db.DateTimeProperty(auto_now=True)

  # indicating wich user last modified the work. Used in displaying Work
  modified_by = db.ReferenceProperty(reference_class=soc.models.user.User,
                                     required=False,
                                     collection_name="modified_comments",
                                     verbose_name=ugettext('Modified by'))

class NewComment(soc.models.base.ModelWithFieldAttributes):
  """Model of a comment on a work.

  A comment is usually associated with a Work or a Proposal,
  for example a Document or a Student Proposal, and with a user, the author.
  There are two types of comment, public (i.e. visible to the student),
  or private (i.e. visible to programme/club staff). Neither type are
  visible to people who are not connected to the work being commented on.
  """

  #: A required many:1 relationship with a comment entity indicating
  #: the user who provided that comment.
  author = db.ReferenceProperty(reference_class=soc.models.role.Profile,
      required=True, collection_name="commented")

  #: The rich textual content of this comment
  content = db.TextProperty(verbose_name='')

  #: Indicated if the comment should be visible to the appropriate student
  is_private = db.BooleanProperty(default=False,
      verbose_name=ugettext('Private'))
  is_private.help_text = ugettext(
      'Whether this comment will only be seen by admins and mentors')

  #: Date when the comment was added
  created = db.DateTimeProperty(auto_now_add=True)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import socket
from lib.config import Config
import time

DO_GPIO = True
try:
    import RPi.GPIO as GPIO
    GPIO.setmode(GPIO.BOARD)
except ModuleNotFoundError:
    DO_GPIO = False

class TallyHandling:

    def __init__(self, source, gpio_port, all_gpios=()):
        self.source = source
        self.state = ''
        self.gpio_port = gpio_port
        if DO_GPIO:
            GPIO.setup(all_gpios, GPIO.OUT)
            GPIO.output(all_gpios, GPIO.HIGH)

    def set_state(self, state):
        self.state = state

    def tally_on(self):
        if DO_GPIO:
            GPIO.output(self.gpio_port, GPIO.LOW)
        print('Tally on')

    def tally_off(self):
        if DO_GPIO:
            GPIO.output(self.gpio_port, GPIO.HIGH)
        print('Tally off')

    def video_change(self, source_a, source_b):
        if self.state == 'fullscreen':
            if source_a == self.source:
                self.tally_on()
            else:
                self.tally_off()
        else:
            if self.source in (source_a, source_b):
                self.tally_on()
            else:
                self.tally_off()

def start_connection(tally_handler):

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.settimeout(2)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

    sock.connect((Config.get('server', 'host'), 9999))
    sock.settimeout(None)

    messages = []
    sock.send(b'get_composite_mode\n')
    sock.send(b'get_video\n')
    sock.send(b'get_stream_status\n')
    while True:
        if len(messages) == 0:
            message = sock.recv(2048)
            message = str(message, 'utf-8')

            if not message:
                start_connection(tally_handler)
                break
            messages = message.split('\n')

        message = messages[0]

        if len(messages) != 0:
            messages = messages[1:]

        if message[:14] == 'composite_mode':
            tally_handler.set_state(message[15:])
        elif message[:12] == 'video_status':
            source_a, source_b = message[13:].split(' ')
            tally_handler.video_change(source_a, source_b)

if __name__ in '__main__':
    try:
        all_gpios = Config.get('light', 'gpios').split(',')
        all_gpios = [int(i) for i in all_gpios]
        tally_handler = TallyHandling(Config.get('light', 'cam'), int(Config.get('light', 'gpio_red')),
                                      all_gpios=all_gpios)

        while True:
            try:
                start_connection(tally_handler)
            except (TimeoutError, ConnectionRefusedError, socket.timeout):
                print('Connection error trying to reconnect in 1s.')
                time.sleep(1)
                continue
    finally:
        print('cleanup')
        if DO_GPIO:
            GPIO.cleanup()
<commit_msg>fixed bug where tallylight would get activated when selecting its camera as source b while in fullscreen composite_mode.<commit_after>#!/usr/bin/env python3
import socket
from lib.config import Config
import time

DO_GPIO = True
try:
    import RPi.GPIO as GPIO
    GPIO.setmode(GPIO.BOARD)
except ModuleNotFoundError:
    DO_GPIO = False

class TallyHandling:

    def __init__(self, source, gpio_port, all_gpios=()):
        self.source = source
        self.state = ''
        self.gpio_port = gpio_port
        if DO_GPIO:
            GPIO.setup(all_gpios, GPIO.OUT)
            GPIO.output(all_gpios, GPIO.HIGH)

    def set_state(self, state):
        self.state = state

    def tally_on(self):
        if DO_GPIO:
            GPIO.output(self.gpio_port, GPIO.LOW)
        print('Tally on')

    def tally_off(self):
        if DO_GPIO:
            GPIO.output(self.gpio_port, GPIO.HIGH)
        print('Tally off')

    def video_change(self, source_a, source_b):
        if self.state == 'fullscreen':
            if source_a == self.source:
                self.tally_on()
            else:
                self.tally_off()
        else:
            if self.source in (source_a, source_b):
                self.tally_on()
            else:
                self.tally_off()

def start_connection(tally_handler):

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.settimeout(2)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)

    sock.connect((Config.get('server', 'host'), 9999))
    sock.settimeout(None)

    messages = []
    sock.send(b'get_composite_mode\n')
    sock.send(b'get_video\n')
    sock.send(b'get_stream_status\n')
    while True:
        if len(messages) == 0:
            message = sock.recv(2048)
            message = str(message, 'utf-8')

            if not message:
                start_connection(tally_handler)
                break
            messages = message.split('\n')

        message = messages[0]

        if len(messages) != 0:
            messages = messages[1:]

        if message[:15] == 'composite_mode ':
            tally_handler.set_state(message[15:])
        elif message[:12] == 'video_status':
            source_a, source_b = message[13:].split(' ')
            tally_handler.video_change(source_a, source_b)

if __name__ in '__main__':
    try:
        all_gpios = Config.get('light', 'gpios').split(',')
        all_gpios = [int(i) for i in all_gpios]
        tally_handler = TallyHandling(Config.get('light', 'cam'), int(Config.get('light', 'gpio_red')),
                                      all_gpios=all_gpios)

        while True:
            try:
                start_connection(tally_handler)
            except (TimeoutError, ConnectionRefusedError, socket.timeout):
                print('Connection error trying to reconnect in 1s.')
                time.sleep(1)
                continue
    finally:
        print('cleanup')
        if DO_GPIO:
            GPIO.cleanup()
<|endoftext|>


<commit_before>from __future__ import print_function

from nose.tools import *
from .utils import *
from unittest import TestCase
try:
    from unittest import mock
except ImportError:
    import mock

import logging
import os
import sys
from tempfile import mkdtemp, TemporaryFile
from shutil import rmtree

import scuba.__main__ as main
import scuba.constants
import scuba

DOCKER_IMAGE = 'debian:8.2'

class TestMain(TestCase):
    def setUp(self):
        # Run each test in its own temp directory
        self.orig_path = os.getcwd()
        self.path = mkdtemp('scubatest')
        os.chdir(self.path)
        logging.info('Temp path: ' + self.path)

    def tearDown(self):
        # Restore the working dir and cleanup the temp one
        rmtree(self.path)
        self.path = None
        os.chdir(self.orig_path)
        self.orig_path = None

    def run_scuba(self, args, exp_retval=0, mock_isatty=False):
        '''Run scuba, checking its return value

        Returns scuba/docker stdout data.
        '''

        # Capture both scuba and docker's stdout/stderr,
        # just as the user would see it.
        # Also mock atexit.register(), so we can simulate file cleanup.

        atexit_funcs = []
        def atexit_reg(cb, *args, **kw):
            atexit_funcs.append((cb, args, kw))

        with TemporaryFile(prefix='scubatest-stdout', mode='w+t') as stdout:
            with TemporaryFile(prefix='scubatest-stderr', mode='w+t') as stderr:
                with mock.patch('atexit.register', side_effect=atexit_reg) as atexit_reg_mock:

                    if mock_isatty:
                        stdout = PseudoTTY(stdout)
                        stderr = PseudoTTY(stderr)

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr

                    sys.stdout = stdout
                    sys.stderr = stderr

                    try:
                        '''
                        Call scuba's main(), and expect it to either exit()
                        with a given return code, or return (implying an exit
                        status of 0).
                        '''
                        try:
                            main.main(argv = args)
                        except SystemExit as sysexit:
                            retcode = sysexit.args[0]
                        else:
                            retcode = 0

                        stdout.seek(0)
                        stderr.seek(0)

                        stdout_data = stdout.read()
                        stderr_data = stderr.read()

                        logging.info('scuba stdout:\n' + stdout_data)
                        logging.info('scuba stderr:\n' + stderr_data)

                        # Verify the return value was as expected
                        assert_equal(exp_retval, retcode)

                        return stdout_data, stderr_data

                    finally:
                        sys.stdout = old_stdout
                        sys.stderr = old_stderr
                        for f, args, kw in atexit_funcs:
                            f(*args, **kw)

    def test_basic(self):
        '''Verify basic scuba functionality'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        args = ['/bin/echo', '-n', 'my output']
        out, _ = self.run_scuba(args)

        assert_str_equalish('my output', out)

    def test_config_error(self):
        '''Verify config errors are handled gracefully'''

        with open('.scuba.yml', 'w') as f:
            f.write('invalid_key: is no good\n')

        # ConfigError -> exit(128)
        self.run_scuba([], 128)

    def test_version(self):
        '''Verify scuba prints its version for -v'''

        out, err = self.run_scuba(['-v'])

        # Argparse in Python < 3.4 printed version to stderr, but
        # changed that to stdout in 3.4. We don't care where it goes.
        # https://bugs.python.org/issue18920
        check = out or err

        assert_startswith(check, 'scuba')

        ver = check.split()[1]
        assert_equal(ver, scuba.__version__)

    def test_no_docker(self):
        '''Verify scuba gracefully handles docker not being installed'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        args = ['/bin/echo', '-n', 'my output']

        old_PATH = os.environ['PATH']
        os.environ['PATH'] = ''

        try:
            _, err = self.run_scuba(args, 2)
        finally:
            os.environ['PATH'] = old_PATH

    @mock.patch('subprocess.call')
    def test_dry_run(self, subproc_call_mock):
        '''Verify scuba handles --dry-run and --verbose'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        args = ['--dry-run', '--verbose', '/bin/false']

        _, err = self.run_scuba(args, 42)

        assert_false(subproc_call_mock.called)

        #TODO: Assert temp files are not cleaned up?

    def test_args(self):
        '''Verify scuba handles cmdline args'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        with open('test.sh', 'w') as f:
            f.write('#!/bin/sh\n')
            f.write('for a in "$@"; do echo $a; done\n')
        make_executable('test.sh')

        lines = ['here', 'are', 'some args']

        out, _ = self.run_scuba(['./test.sh'] + lines)

        assert_seq_equal(out.splitlines(), lines)

    def test_created_file_ownership(self):
        '''Verify files created under scuba have correct ownership'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        filename = 'newfile.txt'

        self.run_scuba(['/bin/touch', filename])

        st = os.stat(filename)
        assert_equal(st.st_uid, os.getuid())
        assert_equal(st.st_gid, os.getgid())

    def _setup_test_tty(self):
        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        with open('check_tty.sh', 'w') as f:
            f.write('#!/bin/sh\n')
            f.write('if [ -t 1 ]; then echo "isatty"; else echo "notatty"; fi\n')
        make_executable('check_tty.sh')

    def test_with_tty(self):
        '''Verify docker allocates tty if stdout is a tty.'''
        self._setup_test_tty()

        out, _ = self.run_scuba(['./check_tty.sh'], mock_isatty=True)

        assert_str_equalish(out, 'isatty')

    def test_without_tty(self):
        '''Verify docker doesn't allocate tty if stdout is not a tty.'''
        self._setup_test_tty()

        out, _ = self.run_scuba(['./check_tty.sh'])

        assert_str_equalish(out, 'notatty')

    def _test_user(self, scuba_args=[]):
        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        args = scuba_args + ['/bin/sh', '-c', 'echo $(id -u) $(id -un) $(id -g) $(id -gn)']
        out, _ = self.run_scuba(args)

        uid, username, gid, groupname = out.split()
        return int(uid), username, int(gid), groupname

    def test_user_scubauser(self):
        '''Verify scuba runs container as the current (host) uid/gid'''

        uid, username, gid, groupname = self._test_user()

        assert_equal(uid, os.getuid())
        assert_equal(username, scuba.constants.SCUBA_USER)
        assert_equal(gid, os.getgid())
        assert_equal(groupname, scuba.constants.SCUBA_GROUP)

    def test_user_root(self):
        '''Verify scuba -r runs container as root'''

        uid, username, gid, groupname = self._test_user(['-r'])

        assert_equal(uid, 0)
        assert_equal(username, 'root')
        assert_equal(gid, 0)
        assert_equal(groupname, 'root')
<commit_msg>Add test for -d (arbitrary docker args)<commit_after>from __future__ import print_function

from nose.tools import *
from .utils import *
from unittest import TestCase
try:
    from unittest import mock
except ImportError:
    import mock

import logging
import os
import sys
from tempfile import mkdtemp, TemporaryFile, NamedTemporaryFile
from shutil import rmtree

import scuba.__main__ as main
import scuba.constants
import scuba

DOCKER_IMAGE = 'debian:8.2'

class TestMain(TestCase):
    def setUp(self):
        # Run each test in its own temp directory
        self.orig_path = os.getcwd()
        self.path = mkdtemp('scubatest')
        os.chdir(self.path)
        logging.info('Temp path: ' + self.path)

    def tearDown(self):
        # Restore the working dir and cleanup the temp one
        rmtree(self.path)
        self.path = None
        os.chdir(self.orig_path)
        self.orig_path = None

    def run_scuba(self, args, exp_retval=0, mock_isatty=False):
        '''Run scuba, checking its return value

        Returns scuba/docker stdout data.
        '''

        # Capture both scuba and docker's stdout/stderr,
        # just as the user would see it.
        # Also mock atexit.register(), so we can simulate file cleanup.

        atexit_funcs = []
        def atexit_reg(cb, *args, **kw):
            atexit_funcs.append((cb, args, kw))

        with TemporaryFile(prefix='scubatest-stdout', mode='w+t') as stdout:
            with TemporaryFile(prefix='scubatest-stderr', mode='w+t') as stderr:
                with mock.patch('atexit.register', side_effect=atexit_reg) as atexit_reg_mock:

                    if mock_isatty:
                        stdout = PseudoTTY(stdout)
                        stderr = PseudoTTY(stderr)

                    old_stdout = sys.stdout
                    old_stderr = sys.stderr

                    sys.stdout = stdout
                    sys.stderr = stderr

                    try:
                        '''
                        Call scuba's main(), and expect it to either exit()
                        with a given return code, or return (implying an exit
                        status of 0).
                        '''
                        try:
                            main.main(argv = args)
                        except SystemExit as sysexit:
                            retcode = sysexit.args[0]
                        else:
                            retcode = 0

                        stdout.seek(0)
                        stderr.seek(0)

                        stdout_data = stdout.read()
                        stderr_data = stderr.read()

                        logging.info('scuba stdout:\n' + stdout_data)
                        logging.info('scuba stderr:\n' + stderr_data)

                        # Verify the return value was as expected
                        assert_equal(exp_retval, retcode)

                        return stdout_data, stderr_data

                    finally:
                        sys.stdout = old_stdout
                        sys.stderr = old_stderr
                        for f, args, kw in atexit_funcs:
                            f(*args, **kw)

    def test_basic(self):
        '''Verify basic scuba functionality'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        args = ['/bin/echo', '-n', 'my output']
        out, _ = self.run_scuba(args)

        assert_str_equalish('my output', out)

    def test_config_error(self):
        '''Verify config errors are handled gracefully'''

        with open('.scuba.yml', 'w') as f:
            f.write('invalid_key: is no good\n')

        # ConfigError -> exit(128)
        self.run_scuba([], 128)

    def test_version(self):
        '''Verify scuba prints its version for -v'''

        out, err = self.run_scuba(['-v'])

        # Argparse in Python < 3.4 printed version to stderr, but
        # changed that to stdout in 3.4. We don't care where it goes.
        # https://bugs.python.org/issue18920
        check = out or err

        assert_startswith(check, 'scuba')

        ver = check.split()[1]
        assert_equal(ver, scuba.__version__)

    def test_no_docker(self):
        '''Verify scuba gracefully handles docker not being installed'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        args = ['/bin/echo', '-n', 'my output']

        old_PATH = os.environ['PATH']
        os.environ['PATH'] = ''

        try:
            _, err = self.run_scuba(args, 2)
        finally:
            os.environ['PATH'] = old_PATH

    @mock.patch('subprocess.call')
    def test_dry_run(self, subproc_call_mock):
        '''Verify scuba handles --dry-run and --verbose'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        args = ['--dry-run', '--verbose', '/bin/false']

        _, err = self.run_scuba(args, 42)

        assert_false(subproc_call_mock.called)

        #TODO: Assert temp files are not cleaned up?

    def test_args(self):
        '''Verify scuba handles cmdline args'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        with open('test.sh', 'w') as f:
            f.write('#!/bin/sh\n')
            f.write('for a in "$@"; do echo $a; done\n')
        make_executable('test.sh')

        lines = ['here', 'are', 'some args']

        out, _ = self.run_scuba(['./test.sh'] + lines)

        assert_seq_equal(out.splitlines(), lines)

    def test_created_file_ownership(self):
        '''Verify files created under scuba have correct ownership'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        filename = 'newfile.txt'

        self.run_scuba(['/bin/touch', filename])

        st = os.stat(filename)
        assert_equal(st.st_uid, os.getuid())
        assert_equal(st.st_gid, os.getgid())

    def _setup_test_tty(self):
        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        with open('check_tty.sh', 'w') as f:
            f.write('#!/bin/sh\n')
            f.write('if [ -t 1 ]; then echo "isatty"; else echo "notatty"; fi\n')
        make_executable('check_tty.sh')

    def test_with_tty(self):
        '''Verify docker allocates tty if stdout is a tty.'''
        self._setup_test_tty()

        out, _ = self.run_scuba(['./check_tty.sh'], mock_isatty=True)

        assert_str_equalish(out, 'isatty')

    def test_without_tty(self):
        '''Verify docker doesn't allocate tty if stdout is not a tty.'''
        self._setup_test_tty()

        out, _ = self.run_scuba(['./check_tty.sh'])

        assert_str_equalish(out, 'notatty')

    def _test_user(self, scuba_args=[]):
        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        args = scuba_args + ['/bin/sh', '-c', 'echo $(id -u) $(id -un) $(id -g) $(id -gn)']
        out, _ = self.run_scuba(args)

        uid, username, gid, groupname = out.split()
        return int(uid), username, int(gid), groupname

    def test_user_scubauser(self):
        '''Verify scuba runs container as the current (host) uid/gid'''

        uid, username, gid, groupname = self._test_user()

        assert_equal(uid, os.getuid())
        assert_equal(username, scuba.constants.SCUBA_USER)
        assert_equal(gid, os.getgid())
        assert_equal(groupname, scuba.constants.SCUBA_GROUP)

    def test_user_root(self):
        '''Verify scuba -r runs container as root'''

        uid, username, gid, groupname = self._test_user(['-r'])

        assert_equal(uid, 0)
        assert_equal(username, 'root')
        assert_equal(gid, 0)
        assert_equal(groupname, 'root')

    def test_arbitrary_docker_args(self):
        '''Verify -d successfully passes arbitrary docker arguments'''

        with open('.scuba.yml', 'w') as f:
            f.write('image: {0}\n'.format(DOCKER_IMAGE))

        data = 'Lorem ipsum dolor sit amet'
        data_path = '/lorem/ipsum'

        with NamedTemporaryFile(mode='wt') as tempf:
            tempf.write(data)
            tempf.flush()

            args = [
                '-d=-v {0}:{1}:ro,z'.format(tempf.name, data_path),
                'cat', data_path,
            ]
            out, _ = self.run_scuba(args)

        assert_str_equalish(out, data)
<|endoftext|>


<commit_before>import os

from simphony.core.cuba import CUBA
from simphony.core.data_container import DataContainer
from simphony.cuds.particles import Particles
from simlammps.io.lammps_data_file_parser import LammpsDataFileParser
from simlammps.io.lammps_simple_data_handler import LammpsSimpleDataHandler
from simlammps.config.domain import get_box

from simlammps.abc_data_manager import ABCDataManager

class LammpsFileIoDataManager(ABCDataManager):
    """  Class managing Lammps data information using file-io

    The class performs communicating the data to and from lammps using FILE-IO
    communications (i.e. through input and output files). The class manages
    data existing in Lammps (via lammps data file) and allows this data to be
    queried and to be changed.

    Class maintains a cache of the particle information.  This information
    is read from file whenever the read() method is called and written to
    the file whenever the flush() method is called.

    """
    def __init__(self):
        super(LammpsFileIoDataManager, self).__init__()

        # map from lammps-id to simphony-uid
        self._lammpsid_to_uid = {}

        # cache of particle containeres
        self._pc_cache = {}

    def get_data(self, uname):
        """Returns data container associated with particle container

        Parameters
        ----------
        uname : string
            non-changing unique name of particles

        """
        return self._pc_cache[uname].data

    def set_data(self, data, uname):
        """Sets data container associated with particle container

        Parameters
        ----------
        uname : string
            non-changing unique name of particles

        """
        self._pc_cache[uname].data = data

    def _handle_delete_particles(self, uname):
        """Handle when a Particles is deleted

        Parameters
        ----------
        uname : string
            non-changing unique name of particles

        """
        del self._pc_cache[uname]

    def _handle_new_particles(self, uname, particles):
        """Add new particle container to this manager.

        Parameters
        ----------
        uname : string
            non-changing unique name of particles
        particles : ABCParticles
            particle container to be added

        """
        # create empty stand-alone particle container
        # to use as a cache of for input/output to LAMMPS
        pc = Particles(name="_")
        pc.data = DataContainer(particles.data)
        for p in particles.iter_particles():
            pc.add_particle(p)
        for p in particles.iter_bonds():
            pc.add_bond(p)

        self._pc_cache[uname] = pc

    def get_particle(self, uid, uname):
        """Get particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        return self._pc_cache[uname].get_particle(uid)

    def update_particle(self, particle, uname):
        """Update particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        return self._pc_cache[uname].update_particle(particle)

    def add_particle(self, particle, uname):
        """Add particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        return self._pc_cache[uname].add_particle(particle)

    def remove_particle(self, uid, uname):
        """Remove particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        self._pc_cache[uname].remove_particle(uid)

    def has_particle(self, uid, uname):
        """Has particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        return self._pc_cache[uname].has_particle(uid)

    def iter_particles(self, uname, uids=None):
        """Iterate over the particles of a certain type

        Parameters
        ----------
        uids : list of particle uids
            sequence of uids of particles that should be iterated over. If
            uids is None then all particles will be iterated over.

        """
        return self._pc_cache[uname].iter_particles(uids)

    def flush(self, input_data_filename):
        """flush to file

        Parameters
        ----------
        input_data_filename :
            name of data-file where inform is written to (i.e lammps's input).
        """
        if self._pc_cache:
            self._write_data_file(input_data_filename)
        else:
            raise RuntimeError(
                "No particles.  Lammps cannot run without a particle")
        # TODO handle properly when there are no particle containers
        # or when some of them do not contain any particles
        # (i.e. someone has deleted all the particles)

    def read(self, output_data_filename):
        """read from file

        Parameters
        ----------
        output_data_filename :
            name of data-file where info read from (i.e lammps's output).
        """
        self._update_from_lammps(output_data_filename)

# Private methods #######################################################
    def _update_from_lammps(self, output_data_filename):
        """read from file and update cache

        """
        assert os.path.isfile(output_data_filename)

        handler = LammpsSimpleDataHandler()
        parser = LammpsDataFileParser(handler)
        parser.parse(output_data_filename)

        atoms = handler.get_atoms()
        velocities = handler.get_velocities()
        masses = handler.get_masses()

        assert(len(atoms) == len(velocities))

        type_data = {}

        for atom_type, mass in masses.iteritems():
            type_data[atom_type] = DataContainer()
            type_data[atom_type][CUBA.MASS] = mass

        # update each particle container with these
        # material-specific attributes
        for _, pc in self._pc_cache.iteritems():
            data = type_data[pc.data[CUBA.MATERIAL_TYPE]]
            for key, value in data.iteritems():
                pc.data[key] = value

        for lammpsid, atom in atoms.iteritems():
            uname, uid = self._lammpsid_to_uid[lammpsid]
            cache_pc = self._pc_cache[uname]
            p = cache_pc.get_particle(uid)
            p.coordinates = tuple(atom[1:4])
            cache_pc.update_particle(p)

            # set the pc's material type
            # (current requirement/assumption is that each
            # pc has particle containers of one type)
            atom_type = atom[0]
            cache_pc.data[CUBA.MATERIAL_TYPE] = atom_type

        for lammpsid, velocity in velocities.iteritems():
            uname, uid = self._lammpsid_to_uid[lammpsid]
            cache_pc = self._pc_cache[uname]
            p = cache_pc.get_particle(uid)
            p.data[CUBA.VELOCITY] = tuple(velocity)
            cache_pc.update_particle(p)

    def _write_data_file(self, filename):
        """ Write data file containing current state of simulation

        """
        lines = ["LAMMPS data file via write_data"
                 ", file written by SimPhony\n\n"]

        # recreate (and store) map from lammps-id to simphony-id
        self._lammpsid_to_uid = {}

        # determine the number of particles
        # and collect the different material types
        # in oder to determine the number of types
        num_particles = 0
        types = set()
        for _, pc in self._pc_cache.iteritems():
            types.add(pc.data[CUBA.MATERIAL_TYPE])
            num_particles += sum(1 for _ in pc.iter_particles())

        box = get_box([pc for _, pc in self._pcs.iteritems()])

        lines.append('{} atoms\n'.format(num_particles))
        lines.append('{} atom types\n\n'.format(len(types)))
        lines.append("\n")
        lines.append(box)
        lines.append("\n")
        lines.append("Masses\n\n")

        material_type_to_mass = self._get_mass()
        for material_type in sorted(material_type_to_mass):
            mass = material_type_to_mass[material_type]
            lines.append('{} {}\n'.format(material_type, mass))
        lines.append("\n")

        with open(filename, 'w') as f:
            f.writelines(lines)

            # keep track of what uid(for a specific pc)
            # corresponds to which lammpsid
            uid_to_lammpsid = {}

            f.write("Atoms\n\n")
            f.writelines(self._generate_atoms_lines(uid_to_lammpsid))

            f.write("\nVelocities\n\n")
            f.writelines(self._generate_velocities_lines(uid_to_lammpsid))
            f.write("\n")

    def _generate_atoms_lines(self, uid_to_lammpsid):
        """ Generate atom lines for a data file.

        Parameters
        ----------
        uid_to_lammpsid : dict
            empty dictionary which will be filled with uname and uid
            of each particle as a key and the corresponding lammpsid
            as the value

        Returns
        -------
        generator
            atom line strings for data file

        """
        lammpsid = 0
        for uname, pc in self._pc_cache.iteritems():
            for p in pc.iter_particles():
                lammpsid += 1
                self._lammpsid_to_uid[lammpsid] = (uname, p.uid)
                uid_to_lammpsid[(uname, p.uid)] = lammpsid
                atom_type = pc.data[CUBA.MATERIAL_TYPE]
                coord = '{0[0]:.16e} {0[1]:.16e} {0[2]:.16e}'.format(
                    p.coordinates)
                yield '{0} {1} {2} 0 0 0\n'.format(
                    lammpsid, atom_type, coord)

    def _generate_velocities_lines(self, uid_to_lammpsid):
        """ Generate velocity lines for a data file.

        Parameters
        ----------
        uid_to_lammpsid : dict
            dictionary filled with uname and uid of each particle as a
            key and the corresponding lammpsid as the value

        Returns
        -------
        generator
            velocities line strings for data file

        """
        for uname, pc in self._pc_cache.iteritems():
            for p in pc.iter_particles():
                lammpsid = uid_to_lammpsid[(uname, p.uid)]
                vel = '{0[0]:.16e} {0[1]:.16e} {0[2]:.16e}'.format(
                    p.data[CUBA.VELOCITY])
                yield '{0} {1}\n'.format(lammpsid, vel)

    def _get_mass(self):
        """ Get a dictionary from 'material type' to 'mass'.

        Check that fits what LAMMPS can handle as well
        as ensure that it works with the limitations
        of how we are currently handling this info.

        Raises
        ------
        RuntimeError :
            if there are particles' which have the same
            material type (CUBA.MATERIAL_TYPE) but different
            masses (CUBA.MASS)

        """
        mass = {}
        for uname, pc in self.iteritems():
            data = pc.data
            material_type = data[CUBA.MATERIAL_TYPE]
            if material_type in mass:
                # check that mass is consistent with an matching type
                if data[CUBA.MASS] != mass[material_type]:
                    raise RuntimeError(
                        "Each material type must have the same mass")
            else:
                mass[material_type] = data[CUBA.MASS]
        return mass
<commit_msg>fix mistake in last commit<commit_after>import os

from simphony.core.cuba import CUBA
from simphony.core.data_container import DataContainer
from simphony.cuds.particles import Particles
from simlammps.io.lammps_data_file_parser import LammpsDataFileParser
from simlammps.io.lammps_simple_data_handler import LammpsSimpleDataHandler
from simlammps.config.domain import get_box

from simlammps.abc_data_manager import ABCDataManager

class LammpsFileIoDataManager(ABCDataManager):
    """  Class managing Lammps data information using file-io

    The class performs communicating the data to and from lammps using FILE-IO
    communications (i.e. through input and output files). The class manages
    data existing in Lammps (via lammps data file) and allows this data to be
    queried and to be changed.

    Class maintains a cache of the particle information.  This information
    is read from file whenever the read() method is called and written to
    the file whenever the flush() method is called.

    """
    def __init__(self):
        super(LammpsFileIoDataManager, self).__init__()

        # map from lammps-id to simphony-uid
        self._lammpsid_to_uid = {}

        # cache of particle containeres
        self._pc_cache = {}

    def get_data(self, uname):
        """Returns data container associated with particle container

        Parameters
        ----------
        uname : string
            non-changing unique name of particles

        """
        return self._pc_cache[uname].data

    def set_data(self, data, uname):
        """Sets data container associated with particle container

        Parameters
        ----------
        uname : string
            non-changing unique name of particles

        """
        self._pc_cache[uname].data = data

    def _handle_delete_particles(self, uname):
        """Handle when a Particles is deleted

        Parameters
        ----------
        uname : string
            non-changing unique name of particles

        """
        del self._pc_cache[uname]

    def _handle_new_particles(self, uname, particles):
        """Add new particle container to this manager.

        Parameters
        ----------
        uname : string
            non-changing unique name of particles
        particles : ABCParticles
            particle container to be added

        """
        # create empty stand-alone particle container
        # to use as a cache of for input/output to LAMMPS
        pc = Particles(name="_")
        pc.data = DataContainer(particles.data)
        for p in particles.iter_particles():
            pc.add_particle(p)
        for p in particles.iter_bonds():
            pc.add_bond(p)

        self._pc_cache[uname] = pc

    def get_particle(self, uid, uname):
        """Get particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        return self._pc_cache[uname].get_particle(uid)

    def update_particle(self, particle, uname):
        """Update particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        self._pc_cache[uname].update_particle(particle)

    def add_particle(self, particle, uname):
        """Add particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        return self._pc_cache[uname].add_particle(particle)

    def remove_particle(self, uid, uname):
        """Remove particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        self._pc_cache[uname].remove_particle(uid)

    def has_particle(self, uid, uname):
        """Has particle

        Parameters
        ----------
        uid :
            uid of particle
        uname : string
            name of particle container

        """
        return self._pc_cache[uname].has_particle(uid)

    def iter_particles(self, uname, uids=None):
        """Iterate over the particles of a certain type

        Parameters
        ----------
        uids : list of particle uids
            sequence of uids of particles that should be iterated over. If
            uids is None then all particles will be iterated over.

        """
        return self._pc_cache[uname].iter_particles(uids)

    def flush(self, input_data_filename):
        """flush to file

        Parameters
        ----------
        input_data_filename :
            name of data-file where inform is written to (i.e lammps's input).
        """
        if self._pc_cache:
            self._write_data_file(input_data_filename)
        else:
            raise RuntimeError(
                "No particles.  Lammps cannot run without a particle")
        # TODO handle properly when there are no particle containers
        # or when some of them do not contain any particles
        # (i.e. someone has deleted all the particles)

    def read(self, output_data_filename):
        """read from file

        Parameters
        ----------
        output_data_filename :
            name of data-file where info read from (i.e lammps's output).
        """
        self._update_from_lammps(output_data_filename)

# Private methods #######################################################
    def _update_from_lammps(self, output_data_filename):
        """read from file and update cache

        """
        assert os.path.isfile(output_data_filename)

        handler = LammpsSimpleDataHandler()
        parser = LammpsDataFileParser(handler)
        parser.parse(output_data_filename)

        atoms = handler.get_atoms()
        velocities = handler.get_velocities()
        masses = handler.get_masses()

        assert(len(atoms) == len(velocities))

        type_data = {}

        for atom_type, mass in masses.iteritems():
            type_data[atom_type] = DataContainer()
            type_data[atom_type][CUBA.MASS] = mass

        # update each particle container with these
        # material-specific attributes
        for _, pc in self._pc_cache.iteritems():
            data = type_data[pc.data[CUBA.MATERIAL_TYPE]]
            for key, value in data.iteritems():
                pc.data[key] = value

        for lammpsid, atom in atoms.iteritems():
            uname, uid = self._lammpsid_to_uid[lammpsid]
            cache_pc = self._pc_cache[uname]
            p = cache_pc.get_particle(uid)
            p.coordinates = tuple(atom[1:4])
            cache_pc.update_particle(p)

            # set the pc's material type
            # (current requirement/assumption is that each
            # pc has particle containers of one type)
            atom_type = atom[0]
            cache_pc.data[CUBA.MATERIAL_TYPE] = atom_type

        for lammpsid, velocity in velocities.iteritems():
            uname, uid = self._lammpsid_to_uid[lammpsid]
            cache_pc = self._pc_cache[uname]
            p = cache_pc.get_particle(uid)
            p.data[CUBA.VELOCITY] = tuple(velocity)
            cache_pc.update_particle(p)

    def _write_data_file(self, filename):
        """ Write data file containing current state of simulation

        """
        lines = ["LAMMPS data file via write_data"
                 ", file written by SimPhony\n\n"]

        # recreate (and store) map from lammps-id to simphony-id
        self._lammpsid_to_uid = {}

        # determine the number of particles
        # and collect the different material types
        # in oder to determine the number of types
        num_particles = 0
        types = set()
        for _, pc in self._pc_cache.iteritems():
            types.add(pc.data[CUBA.MATERIAL_TYPE])
            num_particles += sum(1 for _ in pc.iter_particles())

        box = get_box([pc for _, pc in self._pc_cache.iteritems()])

        lines.append('{} atoms\n'.format(num_particles))
        lines.append('{} atom types\n\n'.format(len(types)))
        lines.append("\n")
        lines.append(box)
        lines.append("\n")
        lines.append("Masses\n\n")

        material_type_to_mass = self._get_mass()
        for material_type in sorted(material_type_to_mass):
            mass = material_type_to_mass[material_type]
            lines.append('{} {}\n'.format(material_type, mass))
        lines.append("\n")

        with open(filename, 'w') as f:
            f.writelines(lines)

            # keep track of what uid(for a specific pc)
            # corresponds to which lammpsid
            uid_to_lammpsid = {}

            f.write("Atoms\n\n")
            f.writelines(self._generate_atoms_lines(uid_to_lammpsid))

            f.write("\nVelocities\n\n")
            f.writelines(self._generate_velocities_lines(uid_to_lammpsid))
            f.write("\n")

    def _generate_atoms_lines(self, uid_to_lammpsid):
        """ Generate atom lines for a data file.

        Parameters
        ----------
        uid_to_lammpsid : dict
            empty dictionary which will be filled with uname and uid
            of each particle as a key and the corresponding lammpsid
            as the value

        Returns
        -------
        generator
            atom line strings for data file

        """
        lammpsid = 0
        for uname, pc in self._pc_cache.iteritems():
            for p in pc.iter_particles():
                lammpsid += 1
                self._lammpsid_to_uid[lammpsid] = (uname, p.uid)
                uid_to_lammpsid[(uname, p.uid)] = lammpsid
                atom_type = pc.data[CUBA.MATERIAL_TYPE]
                coord = '{0[0]:.16e} {0[1]:.16e} {0[2]:.16e}'.format(
                    p.coordinates)
                yield '{0} {1} {2} 0 0 0\n'.format(
                    lammpsid, atom_type, coord)

    def _generate_velocities_lines(self, uid_to_lammpsid):
        """ Generate velocity lines for a data file.

        Parameters
        ----------
        uid_to_lammpsid : dict
            dictionary filled with uname and uid of each particle as a
            key and the corresponding lammpsid as the value

        Returns
        -------
        generator
            velocities line strings for data file

        """
        for uname, pc in self._pc_cache.iteritems():
            for p in pc.iter_particles():
                lammpsid = uid_to_lammpsid[(uname, p.uid)]
                vel = '{0[0]:.16e} {0[1]:.16e} {0[2]:.16e}'.format(
                    p.data[CUBA.VELOCITY])
                yield '{0} {1}\n'.format(lammpsid, vel)

    def _get_mass(self):
        """ Get a dictionary from 'material type' to 'mass'.

        Check that fits what LAMMPS can handle as well
        as ensure that it works with the limitations
        of how we are currently handling this info.

        Raises
        ------
        RuntimeError :
            if there are particles' which have the same
            material type (CUBA.MATERIAL_TYPE) but different
            masses (CUBA.MASS)

        """
        mass = {}
        for uname, pc in self.iteritems():
            data = pc.data
            material_type = data[CUBA.MATERIAL_TYPE]
            if material_type in mass:
                # check that mass is consistent with an matching type
                if data[CUBA.MASS] != mass[material_type]:
                    raise RuntimeError(
                        "Each material type must have the same mass")
            else:
                mass[material_type] = data[CUBA.MASS]
        return mass
<|endoftext|>


<commit_before>try:
    from google.appengine.tools.devappserver2.python import sandbox
except ImportError:
    from google.appengine.tools.devappserver2.python.runtime import sandbox

try:
    from google.appengine.tools.devappserver2.python import runtime
except ImportError:
    from google.appengine.tools.devappserver2.python.runtime import runtime

try:
    from google.appengine.tools.devappserver2.devappserver2 import _LOG_LEVEL_TO_RUNTIME_CONSTANT
except ImportError:
    from google.appengine.tools.devappserver2.constants import LOG_LEVEL_TO_RUNTIME_CONSTANT
    _LOG_LEVEL_TO_RUNTIME_CONSTANT = LOG_LEVEL_TO_RUNTIME_CONSTANT
<commit_msg>Switching the import order, as in sdk 1.9.54 simply attempting to import something from devappserver2 here results in an attribute error when devappserver tries to import mail_stub.<commit_after>try:
    from google.appengine.tools.devappserver2.python import sandbox
except ImportError:
    from google.appengine.tools.devappserver2.python.runtime import sandbox

try:
    from google.appengine.tools.devappserver2.python import runtime
except ImportError:
    from google.appengine.tools.devappserver2.python.runtime import runtime

try:
    from google.appengine.tools.devappserver2.constants import LOG_LEVEL_TO_RUNTIME_CONSTANT
    _LOG_LEVEL_TO_RUNTIME_CONSTANT = LOG_LEVEL_TO_RUNTIME_CONSTANT
except ImportError:
    from google.appengine.tools.devappserver2.devappserver2 import _LOG_LEVEL_TO_RUNTIME_CONSTANT
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import unittest

from language_detector import detect_language, LANGUAGES

class TestLanguageDetector(unittest.TestCase):

    def setUp(self):
        self.languages = [
            {
                'name': 'Spanish',
                'common_words': [
                    'el', 'la', 'de', 'que', 'y', 'a', 'en', 'un', 'ser', 'se',
                    'no', 'haber', 'por', 'con', 'su', 'para', 'como', 'estar',
                    'tener', 'le', 'lo', 'lo', 'todo', 'pero', 'más', 'hacer',
                    'o', 'poder', 'decir', 'este', 'ir', 'otro', 'ese', 'la',
                    'si', 'me', 'ya', 'ver', 'porque', 'dar', 'cuando', 'él',
                    'muy', 'sin', 'vez', 'mucho', 'saber', 'qué', 'sobre',
                    'mi', 'alguno', 'mismo', 'yo', 'también', 'hasta'
                ]
            },
            {
                'name': 'German',
                'common_words': [
                    'das', 'ist', 'du', 'ich', 'nicht', 'die', 'es', 'und',
                    'der', 'was', 'wir', 'zu', 'ein', 'er', 'in', 'sie', 'mir',
                    'mit', 'ja', 'wie', 'den', 'auf', 'mich', 'dass', 'so',
                    'hier', 'eine', 'wenn', 'hat', 'all', 'sind', 'von',
                    'dich', 'war', 'haben', 'für', 'an', 'habe', 'da', 'nein',
                    'bin', 'noch', 'dir', 'uns', 'sich', 'nur',
                    'einen', 'kann', 'dem'
                ]
            }
        ]
        self.texts = {
            "spanish": """
                Lionel Andrés Messi Cuccittini (Rosario, 24 de junio de 1987),
                conocido como Leo Messi, es un futbolista argentino11 que juega
                como delantero en el Fútbol Club Barcelona y en la selección
                argentina, de la que es capitán. Considerado con frecuencia el
                mejor jugador del mundo y calificado en el ámbito deportivo como el
                más grande de todos los tiempos, Messi es el único futbolista en la
                historia que ha ganado cinco veces el FIFA Balón de Oro –cuatro de
                ellos en forma consecutiva– y el primero en
                recibir tres Botas de Oro.
                """,
            "german": """
                Messi spielt seit seinem 14. Lebensjahr für den FC Barcelona.
                Mit 24 Jahren wurde er Rekordtorschütze des FC Barcelona, mit 25
                der jüngste Spieler in der La-Liga-Geschichte, der 200 Tore
                erzielte. Inzwischen hat Messi als einziger Spieler mehr als 300
                Erstligatore erzielt und ist damit Rekordtorschütze
                der Primera División.
                """,
            "mostly-spanish": """
                # spanish
                Lionel Andrés Messi Cuccittini (Rosario, 24 de junio de 1987),
                conocido como Leo Messi, es un futbolista argentino11 que juega
                como delantero en el Fútbol Club Barcelona y en la selección
                argentina, de la que es capitán.

                # german
                Messi spielt seit seinem 14. Lebensjahr für den FC Barcelona.
                Mit 24 Jahren wurde er Rekordtorschütze des FC Barcelona, mit 25
                der jüngste Spieler in der La-Liga-Geschichte, der 200 Tore
                erzielte.
                """
        }

    def test_detect_language_spanish(self):
        result = detect_language(self.texts["spanish"], self.languages)
        self.assertEqual(result, 'Spanish')

    def test_detect_language_german(self):
        result = detect_language(self.texts["german"], self.languages)
        self.assertEqual(result, 'German')

    def test_detect_language_mixed_languages(self):
        result = detect_language(self.texts["mostly-spanish"], self.languages)
        self.assertEqual(result, 'Spanish')
<commit_msg>converted result to lower case before asserting to make the string less rigid<commit_after># -*- coding: utf-8 -*-
import unittest

from language_detector import detect_language, LANGUAGES

class TestLanguageDetector(unittest.TestCase):

    def setUp(self):
        self.languages = [
            {
                'name': 'Spanish',
                'common_words': [
                    'el', 'la', 'de', 'que', 'y', 'a', 'en', 'un', 'ser', 'se',
                    'no', 'haber', 'por', 'con', 'su', 'para', 'como', 'estar',
                    'tener', 'le', 'lo', 'lo', 'todo', 'pero', 'más', 'hacer',
                    'o', 'poder', 'decir', 'este', 'ir', 'otro', 'ese', 'la',
                    'si', 'me', 'ya', 'ver', 'porque', 'dar', 'cuando', 'él',
                    'muy', 'sin', 'vez', 'mucho', 'saber', 'qué', 'sobre',
                    'mi', 'alguno', 'mismo', 'yo', 'también', 'hasta'
                ]
            },
            {
                'name': 'German',
                'common_words': [
                    'das', 'ist', 'du', 'ich', 'nicht', 'die', 'es', 'und',
                    'der', 'was', 'wir', 'zu', 'ein', 'er', 'in', 'sie', 'mir',
                    'mit', 'ja', 'wie', 'den', 'auf', 'mich', 'dass', 'so',
                    'hier', 'eine', 'wenn', 'hat', 'all', 'sind', 'von',
                    'dich', 'war', 'haben', 'für', 'an', 'habe', 'da', 'nein',
                    'bin', 'noch', 'dir', 'uns', 'sich', 'nur',
                    'einen', 'kann', 'dem'
                ]
            }
        ]
        self.texts = {
            "spanish": """
                Lionel Andrés Messi Cuccittini (Rosario, 24 de junio de 1987),
                conocido como Leo Messi, es un futbolista argentino11 que juega
                como delantero en el Fútbol Club Barcelona y en la selección
                argentina, de la que es capitán. Considerado con frecuencia el
                mejor jugador del mundo y calificado en el ámbito deportivo como el
                más grande de todos los tiempos, Messi es el único futbolista en la
                historia que ha ganado cinco veces el FIFA Balón de Oro –cuatro de
                ellos en forma consecutiva– y el primero en
                recibir tres Botas de Oro.
                """,
            "german": """
                Messi spielt seit seinem 14. Lebensjahr für den FC Barcelona.
                Mit 24 Jahren wurde er Rekordtorschütze des FC Barcelona, mit 25
                der jüngste Spieler in der La-Liga-Geschichte, der 200 Tore
                erzielte. Inzwischen hat Messi als einziger Spieler mehr als 300
                Erstligatore erzielt und ist damit Rekordtorschütze
                der Primera División.
                """,
            "mostly-spanish": """
                # spanish
                Lionel Andrés Messi Cuccittini (Rosario, 24 de junio de 1987),
                conocido como Leo Messi, es un futbolista argentino11 que juega
                como delantero en el Fútbol Club Barcelona y en la selección
                argentina, de la que es capitán.

                # german
                Messi spielt seit seinem 14. Lebensjahr für den FC Barcelona.
                Mit 24 Jahren wurde er Rekordtorschütze des FC Barcelona, mit 25
                der jüngste Spieler in der La-Liga-Geschichte, der 200 Tore
                erzielte.
                """
        }

    def test_detect_language_spanish(self):
        result = detect_language(self.texts["spanish"], self.languages)
        self.assertEqual(result.lower(), 'spanish')

    def test_detect_language_german(self):
        result = detect_language(self.texts["german"], self.languages)
        self.assertEqual(result.lower(), 'german')

    def test_detect_language_mixed_languages(self):
        result = detect_language(self.texts["mostly-spanish"], self.languages)
        self.assertEqual(result.lower(), 'spanish')
<|endoftext|>


<commit_before>"""
Implements variance() function.
"""

from .mean import mean
from .decimalize import decimalize

def variance(data, sample=True):
    """
    Variance_ is the sum of squared deviations from the mean. It is a general measurement of
    how far from the mean the values are.

    .. _Variance: http://en.wikipedia.org/wiki/Variance

    Args:
        data: A list of numeric objects.
        sample: A boolean value. If True, calculates sample variance. If False, calculates
            population variance.

    Returns:
        A float object.

    Examples:
        >>> variance([1, 2, 3, 4])
        1.6666666666666667
        >>> variance([1, 2, 3, 4], sample = False)
        1.25
        >>> variance([1, 2, 3, 4, 5, 6])
        3.5
        >>> variance([-2, -1, 0, 1, 2])
        2.5
        >>> variance([1]) # variance of one value is not defined
        >>> variance([4]) # variance of one value is not defined
    """
    if len(data) < 2:
        return(None)

    data = decimalize(data)
    m = decimalize(mean(data))
    if sample: # pylint: disable=no-else-return
        return(float(sum([pow(x - m, 2) for x in data]) / (len(data) - 1)))
    else:
        return(float(mean([pow(x - m, 2) for x in data])))
<commit_msg>Remove unnecessary decimalize from variance().<commit_after>"""
Implements variance() function.
"""

from .mean import mean

def variance(data, sample=True):
    """
    Variance_ is the sum of squared deviations from the mean. It is a general measurement of
    how far from the mean the values are.

    .. _Variance: http://en.wikipedia.org/wiki/Variance

    Args:
        data: A list of numeric objects.
        sample: A boolean value. If True, calculates sample variance. If False, calculates
            population variance.

    Returns:
        A float object.

    Examples:
        >>> variance([1, 2, 3, 4])
        1.6666666666666667
        >>> variance([1, 2, 3, 4], sample = False)
        1.25
        >>> variance([1, 2, 3, 4, 5, 6])
        3.5
        >>> variance([-2, -1, 0, 1, 2])
        2.5
        >>> variance([1]) # variance of one value is not defined
        >>> variance([4]) # variance of one value is not defined
    """
    if len(data) < 2:
        return(None)

    m = mean(data)
    if sample: # pylint: disable=no-else-return
        return(float(sum([pow(x - m, 2) for x in data]) / (len(data) - 1)))
    else:
        return(float(mean([pow(x - m, 2) for x in data])))
<|endoftext|>