<commit_before>"""
Compile a Python script into an executable that embeds CPython and run it.
Requires CPython to be built as a shared library ('libpythonX.Y').

Basic usage:

    python cythonrun somefile.py [ARGS]
"""

DEBUG = True

import sys
import os
from distutils import sysconfig

def get_config_var(name):
    return sysconfig.get_config_var(name) or ''

INCDIR = sysconfig.get_python_inc()
LIBDIR1 = get_config_var('LIBDIR')
LIBDIR2 = get_config_var('LIBPL')
PYLIB = get_config_var('LIBRARY')
PYLIB_DYN = get_config_var('LDLIBRARY')
if PYLIB_DYN == PYLIB:
    # no shared library
    PYLIB_DYN = ''
else:
    PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ

CC = get_config_var('CC')
CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '')
LINKCC = get_config_var('LINKCC')
LINKFORSHARED = get_config_var('LINKFORSHARED')
LIBS = get_config_var('LIBS')
SYSLIBS = get_config_var('SYSLIBS')
EXE_EXT = sysconfig.get_config_var('EXE')

def _debug(msg, *args):
    if DEBUG:
        if args:
            msg = msg % args
        sys.stderr.write(msg + '\n')

def dump_config():
    _debug('INCDIR: %s', INCDIR)
    _debug('LIBDIR1: %s', LIBDIR1)
    _debug('LIBDIR2: %s', LIBDIR2)
    _debug('PYLIB: %s', PYLIB)
    _debug('PYLIB_DYN: %s', PYLIB_DYN)
    _debug('CC: %s', CC)
    _debug('CFLAGS: %s', CFLAGS)
    _debug('LINKCC: %s', LINKCC)
    _debug('LINKFORSHARED: %s', LINKFORSHARED)
    _debug('LIBS: %s', LIBS)
    _debug('SYSLIBS: %s', SYSLIBS)
    _debug('EXE_EXT: %s', EXE_EXT)

def runcmd(cmd, shell=True):
    if shell:
        cmd = ' '.join(cmd)
        _debug(cmd)
    else:
        _debug(' '.join(cmd))

    try:
        import subprocess
    except ImportError: # Python 2.3 ...
        returncode = os.system(cmd)
    else:
        returncode = subprocess.call(cmd, shell=shell)

    if returncode:
        sys.exit(returncode)

def clink(basename):
    runcmd([LINKCC, '-o', basename + EXE_EXT, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2]
           + [PYLIB_DYN and ('-l'+PYLIB_DYN) or os.path.join(LIBDIR1, PYLIB)]
           + LIBS.split() + SYSLIBS.split() + LINKFORSHARED.split())

def ccompile(basename):
    runcmd([CC, '-c', '-o', basename+'.o', basename+'.c', '-I' + INCDIR] + CFLAGS.split())

def cycompile(input_file, options=()):
    from Cython.Compiler import Version, CmdLine, Main
    options, sources = CmdLine.parse_command_line(list(options or ()) + ['--embed', input_file])
    _debug('Using Cython %s to compile %s', Version.version, input_file)
    result = Main.compile(sources, options)
    if result.num_errors > 0:
        sys.exit(1)

def exec_file(program_name, args=()):
    runcmd([os.path.abspath(program_name)] + list(args), shell=False)

def build(input_file, compiler_args=(), force=False):
    """
    Build an executable program from a Cython module.

    Returns the name of the executable file.
    """
    basename = os.path.splitext(input_file)[0]
    exe_file = basename + EXE_EXT
    if (not force and os.path.exists(exe_file) and os.path.exists(input_file)
        and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)):
        _debug("File is up to date, not regenerating %s", exe_file)
        return
    cycompile(input_file, compiler_args)
    ccompile(basename)
    clink(basename)
    return basename + EXE_EXT

def build_and_run(args):
    """
    Build an executable program from a Cython module and runs it.

    Arguments after the module name will be passed verbatimely to the
    program.
    """
    cy_args = []
    last_arg = None
    for i, arg in enumerate(args):
        if arg.startswith('-'):
            cy_args.append(arg)
        elif last_arg in ('-X', '--directive'):
            cy_args.append(arg)
        else:
            input_file = arg
            args = args[i+1:]
            break
        last_arg = arg
    else:
        raise ValueError('no input file provided')

    program_name = build(input_file, cy_args)
    exec_file(program_name, args)

if __name__ == '__main__':
    build_and_run(sys.argv[1:])
<commit_msg>cythonrun: refuse to overwrite input file with output file if both have the same name<commit_after>"""
Compile a Python script into an executable that embeds CPython and run it.
Requires CPython to be built as a shared library ('libpythonX.Y').

Basic usage:

    python cythonrun somefile.py [ARGS]
"""

DEBUG = True

import sys
import os
from distutils import sysconfig

def get_config_var(name):
    return sysconfig.get_config_var(name) or ''

INCDIR = sysconfig.get_python_inc()
LIBDIR1 = get_config_var('LIBDIR')
LIBDIR2 = get_config_var('LIBPL')
PYLIB = get_config_var('LIBRARY')
PYLIB_DYN = get_config_var('LDLIBRARY')
if PYLIB_DYN == PYLIB:
    # no shared library
    PYLIB_DYN = ''
else:
    PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ

CC = get_config_var('CC')
CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '')
LINKCC = get_config_var('LINKCC')
LINKFORSHARED = get_config_var('LINKFORSHARED')
LIBS = get_config_var('LIBS')
SYSLIBS = get_config_var('SYSLIBS')
EXE_EXT = sysconfig.get_config_var('EXE')

def _debug(msg, *args):
    if DEBUG:
        if args:
            msg = msg % args
        sys.stderr.write(msg + '\n')

def dump_config():
    _debug('INCDIR: %s', INCDIR)
    _debug('LIBDIR1: %s', LIBDIR1)
    _debug('LIBDIR2: %s', LIBDIR2)
    _debug('PYLIB: %s', PYLIB)
    _debug('PYLIB_DYN: %s', PYLIB_DYN)
    _debug('CC: %s', CC)
    _debug('CFLAGS: %s', CFLAGS)
    _debug('LINKCC: %s', LINKCC)
    _debug('LINKFORSHARED: %s', LINKFORSHARED)
    _debug('LIBS: %s', LIBS)
    _debug('SYSLIBS: %s', SYSLIBS)
    _debug('EXE_EXT: %s', EXE_EXT)

def runcmd(cmd, shell=True):
    if shell:
        cmd = ' '.join(cmd)
        _debug(cmd)
    else:
        _debug(' '.join(cmd))

    try:
        import subprocess
    except ImportError: # Python 2.3 ...
        returncode = os.system(cmd)
    else:
        returncode = subprocess.call(cmd, shell=shell)

    if returncode:
        sys.exit(returncode)

def clink(basename):
    runcmd([LINKCC, '-o', basename + EXE_EXT, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2]
           + [PYLIB_DYN and ('-l'+PYLIB_DYN) or os.path.join(LIBDIR1, PYLIB)]
           + LIBS.split() + SYSLIBS.split() + LINKFORSHARED.split())

def ccompile(basename):
    runcmd([CC, '-c', '-o', basename+'.o', basename+'.c', '-I' + INCDIR] + CFLAGS.split())

def cycompile(input_file, options=()):
    from Cython.Compiler import Version, CmdLine, Main
    options, sources = CmdLine.parse_command_line(list(options or ()) + ['--embed', input_file])
    _debug('Using Cython %s to compile %s', Version.version, input_file)
    result = Main.compile(sources, options)
    if result.num_errors > 0:
        sys.exit(1)

def exec_file(program_name, args=()):
    runcmd([os.path.abspath(program_name)] + list(args), shell=False)

def build(input_file, compiler_args=(), force=False):
    """
    Build an executable program from a Cython module.

    Returns the name of the executable file.
    """
    basename = os.path.splitext(input_file)[0]
    exe_file = basename + EXE_EXT
    if not force and os.path.abspath(exe_file) == os.path.abspath(input_file):
        raise ValueError("Input and output file names are the same, refusing to overwrite")
    if (not force and os.path.exists(exe_file) and os.path.exists(input_file)
        and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)):
        _debug("File is up to date, not regenerating %s", exe_file)
        return
    cycompile(input_file, compiler_args)
    ccompile(basename)
    clink(basename)
    return basename + EXE_EXT

def build_and_run(args):
    """
    Build an executable program from a Cython module and runs it.

    Arguments after the module name will be passed verbatimely to the
    program.
    """
    cy_args = []
    last_arg = None
    for i, arg in enumerate(args):
        if arg.startswith('-'):
            cy_args.append(arg)
        elif last_arg in ('-X', '--directive'):
            cy_args.append(arg)
        else:
            input_file = arg
            args = args[i+1:]
            break
        last_arg = arg
    else:
        raise ValueError('no input file provided')

    program_name = build(input_file, cy_args)
    exec_file(program_name, args)

if __name__ == '__main__':
    build_and_run(sys.argv[1:])
<|endoftext|>


<commit_before># Copyright (c) 2007 The Hewlett-Packard Development Company
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Gabe Black

microcode = '''
def macroop SYSCALL_64
{
    # All 1s.
    limm t1, "(uint64_t)(-1)", dataSize=8

    # Save the next RIP.
    rdip rcx

    # Stick rflags with RF masked into r11.
    rflags t2
    limm t3, "~RFBit", dataSize=8
    and r11, t2, t3, dataSize=8

    rdval t3, star
    srli t3, t3, 32, dataSize=8
    andi t3, t3, 0xFC, dataSize=1

    # Set up CS.
    wrsel cs, t3
    wrbase cs, t0, dataSize=8
    wrlimit cs, t1, dataSize=4
    # Not writable, read/execute-able, not expandDown,
    # dpl=0, defaultSize=0, long mode
    limm t4, ((0 << 0)  | (0  << 2)  | (0 << 3)   | \
              (1 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (10 << 8)  | (0 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr cs, t4

    # Set up SS.
    addi t3, t3, 8
    wrsel ss, t3
    wrbase ss, t0, dataSize=8
    wrlimit ss, t1, dataSize=4
    # Writable, readable, not expandDown,
    # dpl=0, defaultSize=0, not long mode
    limm t4, ((0 << 0)  | (0  << 2)  | (1 << 3)   | \
              (0 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (2  << 8)  | (1 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr ss, t4

    # Set the new rip.
    rdval t7, lstar
    wrip t0, t7

    # Mask the flags against sf_mask and leave RF turned off.
    rdval t3, sf_mask, dataSize=8
    xor t3, t3, t1, dataSize=8
    and t3, t3, r11, dataSize=8
    wrflags t3, t0
};

def macroop SYSCALL_COMPAT
{
    # All 1s.
    limm t1, "(uint64_t)(-1)", dataSize=8

    # Save the next RIP.
    rdip rcx

    # Stick rflags with RF masked into r11.
    rflags t2
    limm t3, "~RFBit", dataSize=8
    and r11, t2, t3, dataSize=8

    rdval t3, star
    srli t3, t3, 32, dataSize=8
    andi t3, t3, 0xFC, dataSize=1

    # Set up CS.
    wrsel cs, t3
    wrbase cs, t0, dataSize=8
    wrlimit cs, t1, dataSize=4
    # Not writable, read/execute-able, not expandDown,
    # dpl=0, defaultSize=0, long mode
    limm t4, ((0 << 0)  | (0  << 2)  | (0 << 3)   | \
              (1 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (10 << 8)  | (0 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr cs, t4

    # Set up SS.
    addi t3, t3, 8
    wrsel ss, t3
    wrbase ss, t0, dataSize=8
    wrlimit ss, t1, dataSize=4
    # Writable, readable, not expandDown,
    # dpl=0, defaultSize=0, not long mode
    limm t4, ((0 << 0)  | (0  << 2)  | (1 << 3)   | \
              (0 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (2  << 8)  | (1 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr ss, t4

    # Set the new rip.
    rdval t7, cstar
    wrip t0, t7

    # Mask the flags against sf_mask and leave RF turned off.
    rdval t3, sf_mask, dataSize=8
    xor t3, t3, t1, dataSize=8
    and t3, t3, r11, dataSize=8
    wrflags t3, t0
};

def macroop SYSCALL_LEGACY
{
    panic "The syscall instruction isn't implemented in legacy mode."
};

def macroop SYSRET_TO_64
{
    # All 1s.
    limm t1, "(uint64_t)(-1)", dataSize=8

    rdval t3, star
    srli t3, t3, 48, dataSize=8
    ori t3, t3, 3, dataSize=1

    # Set rflags to r11 with RF and VM cleared.
    limm t4, "~(RFBit | VMBit)", dataSize=8
    and t4, t4, r11, dataSize=8
    wrflags t4, t0

    # Set up CS.
    addi t4, t3, 16, dataSize=8
    wrsel cs, t4
    wrbase cs, t0, dataSize=8
    wrlimit cs, t1, dataSize=4
    # Not writable, read/execute-able, not expandDown,
    # dpl=3, defaultSize=0, long mode
    limm t4, ((3 << 0)  | (0  << 2)  | (0 << 3)   | \
              (1 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (10 << 8)  | (0 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr cs, t4

    # Only the selector is changed for SS.
    addi t4, t3, 8, dataSize=8
    wrsel ss, t4

    # Set the RIP back.
    wrip rcx, t0, dataSize=8
};

def macroop SYSRET_TO_COMPAT
{
    # All 1s.
    limm t1, "(uint64_t)(-1)", dataSize=8

    rdval t3, star
    srli t3, t3, 48, dataSize=8
    ori t3, t3, 3, dataSize=1

    # Set rflags to r11 with RF and VM cleared.
    limm t4, "~(RFBit | VMBit)", dataSize=8
    and t4, t4, r11, dataSize=8
    wrflags t4, t0

    # Set up CS.
    wrsel cs, t3
    wrbase cs, t0, dataSize=8
    wrlimit cs, t1, dataSize=4
    # Not writable, read/execute-able, not expandDown,
    # dpl=3, defaultSize=1, not long mode
    limm t4, ((3 << 0)  | (0  << 2)  | (0 << 3)   | \
              (1 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (10 << 8)  | (0 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr cs, t4

    # Only the selector is changed for SS.
    addi t4, t3, 8, dataSize=8
    wrsel ss, t4

    # Set the RIP back.
    wrip rcx, t0, dataSize=8
};

def macroop SYSRET_NON_64
{
    panic "The sysret instruction isn't implemented in legacy mode."
};
'''
#let {{
#    class SYSENTER(Inst):
#       "GenFault ${new UnimpInstFault}"
#    class SYSEXIT(Inst):
#       "GenFault ${new UnimpInstFault}"
#}};
<commit_msg>X86: Use all 64 bits of the lstar register in the SYSCALL_64 macroop.<commit_after># Copyright (c) 2007 The Hewlett-Packard Development Company
# All rights reserved.
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Gabe Black

microcode = '''
def macroop SYSCALL_64
{
    # All 1s.
    limm t1, "(uint64_t)(-1)", dataSize=8

    # Save the next RIP.
    rdip rcx

    # Stick rflags with RF masked into r11.
    rflags t2
    limm t3, "~RFBit", dataSize=8
    and r11, t2, t3, dataSize=8

    rdval t3, star
    srli t3, t3, 32, dataSize=8
    andi t3, t3, 0xFC, dataSize=1

    # Set up CS.
    wrsel cs, t3
    wrbase cs, t0, dataSize=8
    wrlimit cs, t1, dataSize=4
    # Not writable, read/execute-able, not expandDown,
    # dpl=0, defaultSize=0, long mode
    limm t4, ((0 << 0)  | (0  << 2)  | (0 << 3)   | \
              (1 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (10 << 8)  | (0 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr cs, t4

    # Set up SS.
    addi t3, t3, 8
    wrsel ss, t3
    wrbase ss, t0, dataSize=8
    wrlimit ss, t1, dataSize=4
    # Writable, readable, not expandDown,
    # dpl=0, defaultSize=0, not long mode
    limm t4, ((0 << 0)  | (0  << 2)  | (1 << 3)   | \
              (0 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (2  << 8)  | (1 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr ss, t4

    # Set the new rip.
    rdval t7, lstar, dataSize=8
    wrip t0, t7, dataSize=8

    # Mask the flags against sf_mask and leave RF turned off.
    rdval t3, sf_mask, dataSize=8
    xor t3, t3, t1, dataSize=8
    and t3, t3, r11, dataSize=8
    wrflags t3, t0
};

def macroop SYSCALL_COMPAT
{
    # All 1s.
    limm t1, "(uint64_t)(-1)", dataSize=8

    # Save the next RIP.
    rdip rcx

    # Stick rflags with RF masked into r11.
    rflags t2
    limm t3, "~RFBit", dataSize=8
    and r11, t2, t3, dataSize=8

    rdval t3, star
    srli t3, t3, 32, dataSize=8
    andi t3, t3, 0xFC, dataSize=1

    # Set up CS.
    wrsel cs, t3
    wrbase cs, t0, dataSize=8
    wrlimit cs, t1, dataSize=4
    # Not writable, read/execute-able, not expandDown,
    # dpl=0, defaultSize=0, long mode
    limm t4, ((0 << 0)  | (0  << 2)  | (0 << 3)   | \
              (1 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (10 << 8)  | (0 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr cs, t4

    # Set up SS.
    addi t3, t3, 8
    wrsel ss, t3
    wrbase ss, t0, dataSize=8
    wrlimit ss, t1, dataSize=4
    # Writable, readable, not expandDown,
    # dpl=0, defaultSize=0, not long mode
    limm t4, ((0 << 0)  | (0  << 2)  | (1 << 3)   | \
              (0 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (2  << 8)  | (1 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr ss, t4

    # Set the new rip.
    rdval t7, cstar
    wrip t0, t7

    # Mask the flags against sf_mask and leave RF turned off.
    rdval t3, sf_mask, dataSize=8
    xor t3, t3, t1, dataSize=8
    and t3, t3, r11, dataSize=8
    wrflags t3, t0
};

def macroop SYSCALL_LEGACY
{
    panic "The syscall instruction isn't implemented in legacy mode."
};

def macroop SYSRET_TO_64
{
    # All 1s.
    limm t1, "(uint64_t)(-1)", dataSize=8

    rdval t3, star
    srli t3, t3, 48, dataSize=8
    ori t3, t3, 3, dataSize=1

    # Set rflags to r11 with RF and VM cleared.
    limm t4, "~(RFBit | VMBit)", dataSize=8
    and t4, t4, r11, dataSize=8
    wrflags t4, t0

    # Set up CS.
    addi t4, t3, 16, dataSize=8
    wrsel cs, t4
    wrbase cs, t0, dataSize=8
    wrlimit cs, t1, dataSize=4
    # Not writable, read/execute-able, not expandDown,
    # dpl=3, defaultSize=0, long mode
    limm t4, ((3 << 0)  | (0  << 2)  | (0 << 3)   | \
              (1 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (10 << 8)  | (0 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr cs, t4

    # Only the selector is changed for SS.
    addi t4, t3, 8, dataSize=8
    wrsel ss, t4

    # Set the RIP back.
    wrip rcx, t0, dataSize=8
};

def macroop SYSRET_TO_COMPAT
{
    # All 1s.
    limm t1, "(uint64_t)(-1)", dataSize=8

    rdval t3, star
    srli t3, t3, 48, dataSize=8
    ori t3, t3, 3, dataSize=1

    # Set rflags to r11 with RF and VM cleared.
    limm t4, "~(RFBit | VMBit)", dataSize=8
    and t4, t4, r11, dataSize=8
    wrflags t4, t0

    # Set up CS.
    wrsel cs, t3
    wrbase cs, t0, dataSize=8
    wrlimit cs, t1, dataSize=4
    # Not writable, read/execute-able, not expandDown,
    # dpl=3, defaultSize=1, not long mode
    limm t4, ((3 << 0)  | (0  << 2)  | (0 << 3)   | \
              (1 << 4)  | (0  << 5)  | (1 << 6)   | \
              (1 << 7)  | (10 << 8)  | (0 << 12)  | \
              (1 << 13) | (0  << 14) | (1 << 15)), dataSize=8
    wrattr cs, t4

    # Only the selector is changed for SS.
    addi t4, t3, 8, dataSize=8
    wrsel ss, t4

    # Set the RIP back.
    wrip rcx, t0, dataSize=8
};

def macroop SYSRET_NON_64
{
    panic "The sysret instruction isn't implemented in legacy mode."
};
'''
#let {{
#    class SYSENTER(Inst):
#       "GenFault ${new UnimpInstFault}"
#    class SYSEXIT(Inst):
#       "GenFault ${new UnimpInstFault}"
#}};
<|endoftext|>


<commit_before>    # -*- coding: utf-8 -*-

import sys
import klarna

from klarna.const import *

import logging
from Acquisition import aq_inner
from zope.component import getMultiAdapter
from zope.i18nmessageid import MessageFactory
from Products.Five import BrowserView
from zope.component import getUtility
from plone.registry.interfaces import IRegistry
from plone.app.uuid.utils import uuidToURL

#from bda.plone.orders.common import get_order
from bda.plone.payment import (
                               Payment,
                               Payments,
                               )
from bda.plone.payment.interfaces import IPaymentData
from bda.plone.orders.common import OrderData
from bda.plone.shop.interfaces import IShopSettings

from bda.plone.klarnainvoice import IKlarnaInvoiceSettings

_ = MessageFactory('bda.plone.klarnainvoice')
logger = logging.getLogger('bda.plone.invoice')

class KlarnaInvoice(Payment):
    pid = 'klarna'
    label = _('klarnainvoice', 'Klarna Invoice')

    def init_url(self, uid):
        return '%s/@@klarna_invoice?uid=%s' % (self.context.absolute_url(), uid)

class KlarnaInvoicePay(BrowserView):
    """
        uses klarna
        """

    def __call__(self, **kw):
        uid = self.request['uid']
        base_url = self.context.absolute_url()
        registry = getUtility(IRegistry)
        settings = registry.forInterface(IKlarnaInvoiceSettings)

        data = IPaymentData(self.context).data(uid)

        #amount = data['amount']
        #description = data['description']
        #ordernumber = data['ordernumber']
        currency = data['currency']

        #get items for klarna
        order_data = OrderData(self.context, uid)
        order = dict(order_data.order.attrs)

        # Merchant ID
        eid = 2280

        # Shared Secret
        shared_secret = 'qzjaNjloMvifB6z'

        #Initialize the Klarna object
        config = klarna.Config(
            eid=str(eid),
            secret=str(shared_secret),
            country='NO',
            language='NB',
            currency='NOK',
            mode='beta',
            pcstorage='json',
            pcuri='/srv/pclasses.json',
            scheme='https',
            candice=True,
        )

        k = klarna.Klarna(config)
        k.init()

        #Add the cart items
        for booking in order_data.bookings:
            k.add_article(
                        qty = 1,
                        title = 'tittel',
                        price =  100,
                        vat =  25,
                        flags = GoodsIs.INC_VAT)

        import pdb; pdb.set_trace()
        #Add Consumer Information
        addr = klarna.Address(
            email= 'youremail@email.com',
            telno='',
            cellno='40 123 456',
            fname='Testperson-no',
            lname='Approved',
            careof='',
            street='Sæffleberggate 56',
            zip='0563',
            city='Oslo',
            country='NO')

        k.shipping = addr
        k.billing = addr

        ## Set customer IP
        k.clientip = '84.48.92.31'

        (reservation_number, order_status) = k.reserve_amount(
            '011215',
            1,
            pclass=klarna.PClass.Type.INVOICE
        )
        <commit_msg>take invoice settings from cart<commit_after>    # -*- coding: utf-8 -*-

import sys
import klarna

from klarna.const import *

import logging
from Acquisition import aq_inner
from zope.component import getMultiAdapter
from zope.i18nmessageid import MessageFactory
from Products.Five import BrowserView
from zope.component import getUtility
from plone.registry.interfaces import IRegistry
from plone.app.uuid.utils import uuidToURL

#from bda.plone.orders.common import get_order
from bda.plone.payment import (
                               Payment,
                               Payments,
                               )
from bda.plone.payment.interfaces import IPaymentData
from bda.plone.orders.common import OrderData
from bda.plone.shop.interfaces import IShopSettings

from bda.plone.klarnainvoice import IKlarnaInvoiceSettings

_ = MessageFactory('bda.plone.klarnainvoice')
logger = logging.getLogger('bda.plone.invoice')

class KlarnaInvoice(Payment):
    pid = 'klarna'
    label = _('klarnainvoice', 'Klarna Invoice')

    def init_url(self, uid):
        return '%s/@@klarna_invoice?uid=%s' % (self.context.absolute_url(), uid)

class KlarnaInvoicePay(BrowserView):
    """
        uses klarna
        """

    def __call__(self, **kw):
        uid = self.request['uid']
        base_url = self.context.absolute_url()
        registry = getUtility(IRegistry)
        settings = registry.forInterface(IKlarnaInvoiceSettings)

        data = IPaymentData(self.context).data(uid)

        #amount = data['amount']
        #description = data['description']
        #ordernumber = data['ordernumber']
        currency = data['currency']

        #get items for klarna
        order_data = OrderData(self.context, uid)
        order = dict(order_data.order.attrs)

        # Merchant ID
        eid = settings.klarna_eid
        eid = 2280

        # Shared Secret
        shared_secret = settings.klarna_secret
        shared_secret = 'qzjaNjloMvifB6z'

        #other settings from control panel
        terms_uri        =  settings.klarna_terms_uri
        confirmation_uri =  settings.klarna_confirmation_uri

        #Initialize the Klarna object
        config = klarna.Config(
            eid=str(eid),
            secret=str(shared_secret),
            country='NO',
            language='NB',
            currency='NOK',
            mode='beta',
            pcstorage='json',
            pcuri='/srv/pclasses.json',
            scheme='https',
            candice=True,
            pno_encoding=4,
            pno='01122015',

        )

        k = klarna.Klarna(config)
        k.init()

        #Add the cart items
        for booking in order_data.bookings:
            import pdb; pdb.set_trace()
            k.add_article(
                        qty = int(booking.attrs['buyable_count']),
                        title = str(booking.attrs['title']),
                        price = int((booking.attrs.get('net', 0.0)*100)+(booking.attrs.get('net', 0.0)*booking.attrs.get('vat', 0.0))),
                        discount = int((booking.attrs['discount_net'])*100),
                        vat =int(booking.attrs.get('vat', 0.0)*100),
                        flags = GoodsIs.INC_VAT,
        )

        #Add Consumer Information
        addr = klarna.Address(
            email= order['personal_data.email'],
            telno='',
            cellno=order['personal_data.phone'],
            fname=order['personal_data.firstname'],
            lname=order['personal_data.lastname'],
            careof='',
            street='billing_address.street',
            zip='billing_address.zip',
            city='billing_address.city',
            country='NO')

        k.shipping = addr
        k.billing = addr

        ## Set customer IP
        k.clientip = '84.48.92.31'

        (reservation_number, order_status) = k.reserve_amount(
            '070719601',
            Gender.MALE,
            pclass=klarna.PClass.Type.INVOICE,
        )
        <|endoftext|>


<commit_before>from argparse import ArgumentParser
import logging
import pandas as pd
import numpy as np
import itertools
import more_itertools
import os

from functools import partial
from changepoint.mean_shift_model import MeanShiftModel
from changepoint.utils.ts_stats import parallelize_func

__author__ = "Vivek Kulkarni"
__email__ = "viveksck@gmail.com"

import psutil
from multiprocessing import cpu_count

p = psutil.Process(os.getpid())
p.set_cpu_affinity(list(range(cpu_count())))

# Global variable specifying which column index the time series
# begins in a dataframe
TS_OFFSET = 2

LOGFORMAT = "%(asctime).19s %(levelname)s %(filename)s: %(lineno)s %(message)s"

def normalize_timeseries(df):
    """
        Normalize each column of the data frame by its mean and standard
        deviation.
    """
    dfm = df.copy(deep=True)
    dfmean = df.mean()
    dfstd = df.std()
    for col in df.columns[2:]:
        dfm[col] = (df[col] - dfmean[col]) / dfstd[col]
    return dfm

def get_filtered_df(df, vocab_file):
    """ Return a data frame with only the words present in the vocab file. """
    if vocab_file:
        vocab = open(vocab_file).readlines()
        vocab = [v.strip() for v in vocab]
        # Get the set of words.
        words = pd.Series(df.word.values.ravel()).unique()
        set_words = set(words)
        # Find the words common to data frame and vocab
        common_set_words = set_words & set(vocab)
        # Filter the dataframe
        df_filtered = df[df.word.isin(common_set_words)]
        return df_filtered
    else:
        return df

def get_pval_word(df, word, B):
    """
    Get the pvalue of a change point at each time point 't' corresponding to
    the word. Also return the number of tail successes during boot strap.
    Use a mean shift model for this.
    """
    # Remove the first TS_OFFSET columns as it is 'index' and 'word' to get the
    # time series for that word.
    ts = df[df.word == word].values[0][TS_OFFSET:]
    # Create a mean shift model
    model = MeanShiftModel()
    # Detect the change points using a mean shift model
    stats_ts, pvals, nums = model.detect_mean_shift(ts, B=B)
    # Return the word and pvals associated with each time point.
    L = [word]
    L.extend(pvals)
    H = [word]
    H.extend(nums)
    return L, H

def get_pval_word_chunk(chunk, df, B):
    """ Get the p-values for each time point for a chunk of words. """
    results = [get_pval_word(df, w, B) for w in chunk]
    return results

def get_minpval_cp(pvalue_df_row):
    """
    Get the minimum p-value and the corresponding time point for each word.
    """
    # first column is 'word', so ignore it
    index_series = pvalue_df_row.index[1:]
    row_series = pvalue_df_row.values[1:]
    assert(len(index_series) == len(row_series))

    # Find the minimum pvalue
    min_pval = np.min(row_series)
    # Find the index where the minimum pvalue occurrs.
    min_idx = np.argmin(row_series)
    # Get the timepoint corresponding to that index
    min_cp = index_series[min_idx]

    return min_pval, min_cp

def get_cp_pval(pvalue_df_row, zscore_df, threshold=0.0):
    """
        Get the minimum p-value corresponding timepoint which also has
        a Z-SCORE > threshold.

    """
    # First column is 'word', so ignore it
    row_series = pvalue_df_row.values[1:]
    # Corresponding Z-Score series for the exact same set of timepoints.
    zscore_series = np.array(zscore_df[zscore_df.word == pvalue_df_row.word][pvalue_df_row.index[1:]])[0]
    assert(len(zscore_series) == len(row_series))

    # Get all the indices where zscore exceeds a threshold
    sel_idx = np.where(zscore_series > threshold)[0]
    # If there are no such indices return NAN
    if not len(sel_idx):
        return 1.0, np.nan

    # We have some indices. Select the pvalues for those indices.
    pvals_indices = np.take(row_series, sel_idx)
    # Find the minimum pvalue among those candidates.
    min_pval = np.min(pvals_indices)
    # Find the minimum candidate index corresponding to that pvalue
    min_idx = np.argmin(pvals_indices)
    # Select the actual index that it corresponds to
    cp_idx = sel_idx[min_idx]
    # Translate that to the actual timepoint and return it.
    cp = pvalue_df_row.index[1:][cp_idx]
    return min_pval, cp

def main(args):
    # Read the arguments
    df_f = args.filename
    common_vocab_file = args.vocab_file
    pval_file = args.pval_file
    sample_file = args.sample_file
    col_to_drop = args.col
    should_normalize = not(args.dont_normalize)
    threshold = float(args.threshold)

    workers = args.workers

    print "Config:"
    print "Input data frame file name:", df_f
    print "Vocab file", common_vocab_file
    print "Output pvalue file", pval_file
    print "Output sample file", sample_file
    print "Columns to drop", col_to_drop
    print "Normalize Time series:", should_normalize
    print "Threshold", threshold

    # Read the time series data
    df = pd.read_csv(df_f)
    # Consider only words in the common vocabulary.
    df = get_filtered_df(df, common_vocab_file)

    # Normalize the data frame
    if should_normalize:
        norm_df = normalize_timeseries(df)
    else:
        norm_df = df

    # Drop the column if needed. We typically drop the 1st column as it always is 0 by
    # default.
    if col_to_drop in norm_df.columns:
        cols = df.columns.tolist()
        if col_to_drop == norm_df.columns[-1]:
            time_points = cols[2:]
            new_cols = cols[0:2] + time_points[::-1]
            norm_df = norm_df[new_cols]
        norm_df.drop(col_to_drop, axis=1, inplace=True)
        print "Dropped column", col_to_drop

    print "Columns of the data frame are", norm_df.columns
    cwords = norm_df.word.values
    print "Number of words we are analyzing:", len(cwords)

    chunksz = np.ceil(len(cwords) / float(workers))
    results = parallelize_func(cwords[:], get_pval_word_chunk, chunksz=chunksz, n_jobs=workers, df=norm_df, B=args.B)

    pvals, num_samples = zip(*results)

    header = ['word'] + list(norm_df.columns[TS_OFFSET:len(pvals[0]) + 1])
    pvalue_df = pd.DataFrame().from_records(list(pvals), columns=header)

    # Append additonal columns to the final df
    pvalue_df_final = pvalue_df.copy(deep=True)

    pvalue_df_final['min_pval'], pvalue_df_final['cp'] = zip(*pvalue_df.apply(get_minpval_cp, axis=1))
    pvalue_df_final['tpval'], pvalue_df_final['tcp'] = zip(*pvalue_df.apply(get_cp_pval, axis=1, zscore_df=norm_df, threshold=threshold))

    # Write the pvalue output.
    num_samples_df = pd.DataFrame().from_records(list(num_samples), columns=header)
    num_samples_df.to_csv(sample_file, encoding='utf-8')

    # Write the sample output
    sdf = pvalue_df_final.sort(columns=['tpval'])
    sdf.to_csv(pval_file, encoding='utf-8')

if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument("-f", "--file", dest="filename", help="Input time series file")
    parser.add_argument("-v", "--vfile", dest="vocab_file", help="Common Vocab file")
    parser.add_argument("-p", "--pfile", dest="pval_file", help="Output pvalue file")
    parser.add_argument("-n", "--nfile", dest="sample_file", help="Output sample file")
    parser.add_argument("-c", "--col", dest="col", help="column to drop")
    parser.add_argument("-d", "--dont_normalize", dest="dont_normalize", action='store_true', default=False, help="Dont normalize")
    parser.add_argument("-t", "--threshold", dest="threshold", default=1.75, type=float, help="Threshold to use for mean shift model.")
    parser.add_argument("-b", "--bootstrap", dest="B", default=1000, type=int, help="Number of  bootstrapped samples to take(default:1000)")
    parser.add_argument("-w", "--workers", dest="workers", default=1, type=int, help="Number of workers to use")
    parser.add_argument("-l", "--log", dest="log", help="log verbosity level", default="INFO")
    args = parser.parse_args()
    if args.log == 'DEBUG':
        sys.excepthook = debug
    numeric_level = getattr(logging, args.log.upper(), None)
    logging.basicConfig(level=numeric_level, format=LOGFORMAT)
    main(args)
<commit_msg>Only dumping relevant pval columns<commit_after>from argparse import ArgumentParser
import logging
import pandas as pd
import numpy as np
import itertools
import more_itertools
import os

from functools import partial
from changepoint.mean_shift_model import MeanShiftModel
from changepoint.utils.ts_stats import parallelize_func

__author__ = "Vivek Kulkarni"
__email__ = "viveksck@gmail.com"

import psutil
from multiprocessing import cpu_count

p = psutil.Process(os.getpid())
p.set_cpu_affinity(list(range(cpu_count())))

# Global variable specifying which column index the time series
# begins in a dataframe
TS_OFFSET = 2

LOGFORMAT = "%(asctime).19s %(levelname)s %(filename)s: %(lineno)s %(message)s"

def normalize_timeseries(df):
    """
        Normalize each column of the data frame by its mean and standard
        deviation.
    """
    dfm = df.copy(deep=True)
    dfmean = df.mean()
    dfstd = df.std()
    for col in df.columns[2:]:
        dfm[col] = (df[col] - dfmean[col]) / dfstd[col]
    return dfm

def get_filtered_df(df, vocab_file):
    """ Return a data frame with only the words present in the vocab file. """
    if vocab_file:
        vocab = open(vocab_file).readlines()
        vocab = [v.strip() for v in vocab]
        # Get the set of words.
        words = pd.Series(df.word.values.ravel()).unique()
        set_words = set(words)
        # Find the words common to data frame and vocab
        common_set_words = set_words & set(vocab)
        # Filter the dataframe
        df_filtered = df[df.word.isin(common_set_words)]
        return df_filtered
    else:
        return df

def get_pval_word(df, word, B):
    """
    Get the pvalue of a change point at each time point 't' corresponding to
    the word. Also return the number of tail successes during boot strap.
    Use a mean shift model for this.
    """
    # Remove the first TS_OFFSET columns as it is 'index' and 'word' to get the
    # time series for that word.
    ts = df[df.word == word].values[0][TS_OFFSET:]
    # Create a mean shift model
    model = MeanShiftModel()
    # Detect the change points using a mean shift model
    stats_ts, pvals, nums = model.detect_mean_shift(ts, B=B)
    # Return the word and pvals associated with each time point.
    L = [word]
    L.extend(pvals)
    H = [word]
    H.extend(nums)
    return L, H

def get_pval_word_chunk(chunk, df, B):
    """ Get the p-values for each time point for a chunk of words. """
    results = [get_pval_word(df, w, B) for w in chunk]
    return results

def get_minpval_cp(pvalue_df_row):
    """
    Get the minimum p-value and the corresponding time point for each word.
    """
    # first column is 'word', so ignore it
    index_series = pvalue_df_row.index[1:]
    row_series = pvalue_df_row.values[1:]
    assert(len(index_series) == len(row_series))

    # Find the minimum pvalue
    min_pval = np.min(row_series)
    # Find the index where the minimum pvalue occurrs.
    min_idx = np.argmin(row_series)
    # Get the timepoint corresponding to that index
    min_cp = index_series[min_idx]

    return min_pval, min_cp

def get_cp_pval(pvalue_df_row, zscore_df, threshold=0.0):
    """
        Get the minimum p-value corresponding timepoint which also has
        a Z-SCORE > threshold.

    """
    # First column is 'word', so ignore it
    row_series = pvalue_df_row.values[1:]
    # Corresponding Z-Score series for the exact same set of timepoints.
    zscore_series = np.array(zscore_df[zscore_df.word == pvalue_df_row.word][pvalue_df_row.index[1:]])[0]
    assert(len(zscore_series) == len(row_series))

    # Get all the indices where zscore exceeds a threshold
    sel_idx = np.where(zscore_series > threshold)[0]
    # If there are no such indices return NAN
    if not len(sel_idx):
        return 1.0, np.nan

    # We have some indices. Select the pvalues for those indices.
    pvals_indices = np.take(row_series, sel_idx)
    # Find the minimum pvalue among those candidates.
    min_pval = np.min(pvals_indices)
    # Find the minimum candidate index corresponding to that pvalue
    min_idx = np.argmin(pvals_indices)
    # Select the actual index that it corresponds to
    cp_idx = sel_idx[min_idx]
    # Translate that to the actual timepoint and return it.
    cp = pvalue_df_row.index[1:][cp_idx]
    return min_pval, cp

def main(args):
    # Read the arguments
    df_f = args.filename
    common_vocab_file = args.vocab_file
    pval_file = args.pval_file
    sample_file = args.sample_file
    col_to_drop = args.col
    should_normalize = not(args.dont_normalize)
    threshold = float(args.threshold)

    workers = args.workers

    print "Config:"
    print "Input data frame file name:", df_f
    print "Vocab file", common_vocab_file
    print "Output pvalue file", pval_file
    print "Output sample file", sample_file
    print "Columns to drop", col_to_drop
    print "Normalize Time series:", should_normalize
    print "Threshold", threshold

    # Read the time series data
    df = pd.read_csv(df_f)
    # Consider only words in the common vocabulary.
    df = get_filtered_df(df, common_vocab_file)

    # Normalize the data frame
    if should_normalize:
        norm_df = normalize_timeseries(df)
    else:
        norm_df = df

    # Drop the column if needed. We typically drop the 1st column as it always is 0 by
    # default.
    if col_to_drop in norm_df.columns:
        cols = df.columns.tolist()
        if col_to_drop == norm_df.columns[-1]:
            time_points = cols[2:]
            new_cols = cols[0:2] + time_points[::-1]
            norm_df = norm_df[new_cols]
        norm_df.drop(col_to_drop, axis=1, inplace=True)
        print "Dropped column", col_to_drop

    print "Columns of the data frame are", norm_df.columns
    cwords = norm_df.word.values
    print "Number of words we are analyzing:", len(cwords)

    chunksz = np.ceil(len(cwords) / float(workers))
    results = parallelize_func(cwords[:], get_pval_word_chunk, chunksz=chunksz, n_jobs=workers, df=norm_df, B=args.B)

    pvals, num_samples = zip(*results)

    header = ['word'] + list(norm_df.columns[TS_OFFSET:len(pvals[0]) + 1])
    pvalue_df = pd.DataFrame().from_records(list(pvals), columns=header)

    # Append additonal columns to the final df
    pvalue_df_final = pvalue_df.copy(deep=True)

    pvalue_df_final['min_pval'], pvalue_df_final['cp'] = zip(*pvalue_df.apply(get_minpval_cp, axis=1))
    pvalue_df_final['tpval'], pvalue_df_final['tcp'] = zip(*pvalue_df.apply(get_cp_pval, axis=1, zscore_df=norm_df, threshold=threshold))

    pvalue_df_final.drop(norm_df.columns[TS_OFFSET:len(pvals[0]) + 1], axis=1, inplace = True)

    # Write the pvalue output.
    num_samples_df = pd.DataFrame().from_records(list(num_samples), columns=header)
    num_samples_df.to_csv(sample_file, encoding='utf-8')

    # Write the sample output
    sdf = pvalue_df_final.sort(columns=['tpval'])
    sdf.to_csv(pval_file, encoding='utf-8')

if __name__ == "__main__":
    parser = ArgumentParser()
    parser.add_argument("-f", "--file", dest="filename", help="Input time series file")
    parser.add_argument("-v", "--vfile", dest="vocab_file", help="Common Vocab file")
    parser.add_argument("-p", "--pfile", dest="pval_file", help="Output pvalue file")
    parser.add_argument("-n", "--nfile", dest="sample_file", help="Output sample file")
    parser.add_argument("-c", "--col", dest="col", help="column to drop")
    parser.add_argument("-d", "--dont_normalize", dest="dont_normalize", action='store_true', default=False, help="Dont normalize")
    parser.add_argument("-t", "--threshold", dest="threshold", default=1.75, type=float, help="Threshold to use for mean shift model.")
    parser.add_argument("-b", "--bootstrap", dest="B", default=1000, type=int, help="Number of  bootstrapped samples to take(default:1000)")
    parser.add_argument("-w", "--workers", dest="workers", default=1, type=int, help="Number of workers to use")
    parser.add_argument("-l", "--log", dest="log", help="log verbosity level", default="INFO")
    args = parser.parse_args()
    if args.log == 'DEBUG':
        sys.excepthook = debug
    numeric_level = getattr(logging, args.log.upper(), None)
    logging.basicConfig(level=numeric_level, format=LOGFORMAT)
    main(args)
<|endoftext|>


<commit_before># coding: utf-8
#
# Copyright 2016 The MyCleanIndia Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#from __future__ import print_function
from django.template.response import TemplateResponse

def index(request):
    """
    Args: request.
    Returns: default application home view.
    """
    return TemplateResponse(request, 'app/home.html',)<commit_msg>Fixed redirect issue.<commit_after># coding: utf-8
#
# Copyright 2016 The MyCleanIndia Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

#from __future__ import print_function
from django.template.response import TemplateResponse

def index(request):
    """
    Args: request.
    Returns: default application home view.
    """
    if request.user.is_authenticated():
    	return TemplateResponse(request, 'app/app.html',)
    return TemplateResponse(request, 'app/home.html',)<|endoftext|>


<commit_before>from django.shortcuts import render, render_to_response
from home.models import Users,Grade,Constants
from django.template import Context, Template
from django.http import HttpResponseRedirect
from django.http import HttpResponse
from django.contrib import messages
from restaurant.models import Restaurant
from home.forms import UserForm
import logging
import requests

def home(request):
    flag = hava()
    return render(request, 'home/home.html')

def statistics(request):
     return render(request, 'home/statistics.html')

def grading(request):
     users = Users.objects.values()
     users_all = Users.objects.all()
     flag = True
     for u in users_all:
         u_temp = Grade.objects.filter(user_id = u.id)
         if not u_temp:
             flag = False

     return render(request, 'home/grading.html',Context({'IsGradingDone':flag,'period':Constants.objects.filter(name = 'period'),'Users':users}))

def users(request):
    try:
        users = Users.objects.values()
        return render(request, 'home/users.html',Context({'Users':users}))
    except Users.DoesNotExist:
        return render(request, 'home/users.html')
def deleteUser(request, id):
    query = Users.objects.get(pk=id)
    query.delete()
    return HttpResponseRedirect('/users/')

def addUser(request):
    if request.POST:
        form = UserForm(request.POST)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect('/users/')
        else:
            form = RetaurantForm()
            return HttpResponseRedirect('/users/')

def gradeIt(request):
    if request.method == 'POST':
        clickedUserId = request.POST.get('user_id')
        rests = Restaurant.objects.values()
        return render(request, 'home/gradeIt.html',Context({'user':Users.objects.get(id=clickedUserId),'Restaurant':rests}))

def enterPeriod(request):
    if request.method == 'POST':
        entered_period = request.POST.get('period')
        checkperiod = Constants.objects.filter(name = 'period')
        if checkperiod.exists():
            Constants.objects.get(name = 'period').update(value = entered_period)
        else:
            newCons = Constants(name = 'period', value = entered_period)
            newCons.save()

        return HttpResponseRedirect('/grading/')

def editPeriod(request):
    if  request.method == 'POST':
        Grade.objects.all().delete()
        Constants.objects.filter(name = 'period').delete()
        return HttpResponseRedirect('/grading/')

def saveGrades(request):
    if  request.method == 'POST':
         grades = request.POST.getlist('grade[]')
         rids = request.POST.getlist('rid[]')
         uid = request.POST.get('uid')
         userGrade =  Grade.objects.filter(user_id= uid)
         counter = len(grades)
         i=0

         if userGrade.exists():
             while i < counter:
                 Grade.objects.select_related().filter(user_id = uid, rest_id = rids[i]).update(grade = grades[i])
                 i = i + 1
         else:
             while i < counter:
                 newGrade = Grade(rest_id = rids[i], user_id = uid, grade = grades[i])
                 newGrade.save()
                 i = i + 1

    return HttpResponseRedirect('/grading/')

def hava():
        r = requests.get('http://api.openweathermap.org/data/2.5/weather?q=Istanbul&APPID=b9dd3952f36a165aecc5518e9e0a5117')
        deneme = r.json()
        tempKelvin = deneme['main']['temp']
        tempCelcius = tempKelvin - 273,15
        humidity = deneme['main']['humidity']
        weatherGroup = deneme['weather'][0]['main']
        weatherId = deneme['weather'][0]['id']
        if (weatherId >= 900 or (weatherId >= 600 and weatherId <=699 ) or (weatherId >= 200 and weatherId <= 299 ) or ( weatherId >= 501 and weatherId <=599 ) or ( weatherId >= 312 and weatherId <= 321 ) and weatherId == 302 ):
            return False
        return True

<commit_msg>al işte al<commit_after>from django.shortcuts import render, render_to_response
from home.models import Users,Grade,Constants
from django.template import Context, Template
from django.http import HttpResponseRedirect
from django.http import HttpResponse
from django.contrib import messages
from restaurant.models import Restaurant
from home.forms import UserForm
import logging
import requests

def home(request):
    flag = hava()
    return render(request, 'home/home.html')

def statistics(request):
     return render(request, 'home/statistics.html')

def grading(request):
     users = Users.objects.values()
     users_all = Users.objects.all()
     flag = True
     for u in users_all:
         u_temp = Grade.objects.filter(user_id = u.id)
         if not u_temp:
             flag = False

     return render(request, 'home/grading.html',Context({'IsGradingDone':flag,'period':Constants.objects.filter(name = 'period'),'Users':users}))

def users(request):
    try:
        users = Users.objects.values()
        return render(request, 'home/users.html',Context({'Users':users}))
    except Users.DoesNotExist:
        return render(request, 'home/users.html')
def deleteUser(request, id):
    query = Users.objects.get(pk=id)
    query.delete()
    return HttpResponseRedirect('/users/')

def addUser(request):
    if request.POST:
        form = UserForm(request.POST)
        if form.is_valid():
            form.save()
            return HttpResponseRedirect('/users/')
        else:
            form = UserForm()
            return HttpResponseRedirect('/users/')

def gradeIt(request):
    if request.method == 'POST':
        clickedUserId = request.POST.get('user_id')
        rests = Restaurant.objects.values()
        return render(request, 'home/gradeIt.html',Context({'user':Users.objects.get(id=clickedUserId),'Restaurant':rests}))

def enterPeriod(request):
    if request.method == 'POST':
        entered_period = request.POST.get('period')
        checkperiod = Constants.objects.filter(name = 'period')
        if checkperiod.exists():
            Constants.objects.get(name = 'period').update(value = entered_period)
        else:
            newCons = Constants(name = 'period', value = entered_period)
            newCons.save()

        return HttpResponseRedirect('/grading/')

def editPeriod(request):
    if  request.method == 'POST':
        Grade.objects.all().delete()
        Constants.objects.filter(name = 'period').delete()
        return HttpResponseRedirect('/grading/')

def saveGrades(request):
    if  request.method == 'POST':
         grades = request.POST.getlist('grade[]')
         rids = request.POST.getlist('rid[]')
         uid = request.POST.get('uid')
         userGrade =  Grade.objects.filter(user_id= uid)
         counter = len(grades)
         i=0

         if userGrade.exists():
             while i < counter:
                 Grade.objects.select_related().filter(user_id = uid, rest_id = rids[i]).update(grade = grades[i])
                 i = i + 1
         else:
             while i < counter:
                 newGrade = Grade(rest_id = rids[i], user_id = uid, grade = grades[i])
                 newGrade.save()
                 i = i + 1

    return HttpResponseRedirect('/grading/')

def hava():
        r = requests.get('http://api.openweathermap.org/data/2.5/weather?q=Istanbul&APPID=b9dd3952f36a165aecc5518e9e0a5117')
        deneme = r.json()
        tempKelvin = deneme['main']['temp']
        tempCelcius = tempKelvin - 273,15
        humidity = deneme['main']['humidity']
        weatherGroup = deneme['weather'][0]['main']
        weatherId = deneme['weather'][0]['id']
        if (weatherId >= 900 or (weatherId >= 600 and weatherId <=699 ) or (weatherId >= 200 and weatherId <= 299 ) or ( weatherId >= 501 and weatherId <=599 ) or ( weatherId >= 312 and weatherId <= 321 ) and weatherId == 302 ):
            return False
        return True

<|endoftext|>


<commit_before>import os
import datetime
import subprocess
import multiprocessing
import math
try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId
from django.core.management.base import BaseCommand, CommandError
from gnowsys_ndf.ndf.models import node_collection, triple_collection, filehive_collection, counter_collection
from gnowsys_ndf.ndf.models import HistoryManager
from gnowsys_ndf.settings import GSTUDIO_DATA_ROOT, GSTUDIO_LOGS_DIR_PATH, MEDIA_ROOT
from schema_mapping import create_factory_schema_mapper

node_collection_ids = set()
triple_collection_ids = set()
filehives_collection_ids = set()
counter_collection_ids = set()
filehives_media_urls = set()
build_rcs = set()
rcs_paths_found = set()
media_url_not_found = set()
media_url_found = set()
historyMgr = HistoryManager()
data_export_path = None
media_export_path = None
log_file = None

def create_log_file(dump_path):
    '''
        Creates log file in gstudio-logs/ with
        the name of the dump folder
    '''
    dump_path = dump_path.split("/")[-1]
    log_file_name = 'group_dump_' + str(dump_path)+ '.log'
    if not os.path.exists(GSTUDIO_LOGS_DIR_PATH):
        os.makedirs(GSTUDIO_LOGS_DIR_PATH)

    log_file_path = os.path.join(GSTUDIO_LOGS_DIR_PATH, log_file_name)
    # print log_file_path
    global log_file
    log_file = open(log_file_path, 'w+')
    log_file.write("\n######### Script ran on : " + str(datetime.datetime.now()) + " #########\n\n")
    return log_file_path

def setup_dump_path(group_name):
    '''
        Creates factory_schema.json which will hold basic info
        like ObjectId, name, type of TYPES_LIST and GSTUDIO_DEFAULT_GROUPS
    '''
    global data_export_path
    global media_export_path
    datetimestamp = datetime.datetime.now().isoformat()
    data_export_path = os.path.join(GSTUDIO_DATA_ROOT, 'data_export', group_name + "_" + str(datetimestamp))
    media_export_path = os.path.join(data_export_path, 'media_files')
    if not os.path.exists(data_export_path):
        os.makedirs(data_export_path)
    if not os.path.exists(media_export_path):
        os.makedirs(media_export_path)
    return data_export_path

def get_triple_data(node_id):
    '''
    Gets all data stored in triples for this node.
    Fetches GAttrtibutes as wells as GRelations.
    '''
    triple_query = {"_type": {'$in': ["GAttribute", "GRelation"]}, "subject": node_id}

    node_gattr_grel_cur = triple_collection.find(triple_query)
    fetch_value = None
    if node_gattr_grel_cur:

        for each_triple_node in node_gattr_grel_cur:
            dump_node(node=each_triple_node,
                collection_name=triple_collection)
            # Get ObjectIds in object_value fields
            if each_triple_node._type is "GAttribute":
                fetch_value = "object_value"
            elif each_triple_node._type is "GRelation":
                fetch_value = "right_subject"
            if fetch_value:
                print "\n fetch_value -- ", fetch_value
                if type(each_triple_node[fetch_value]) == list and all(isinstance(each_obj_value, ObjectId) for each_obj_value in each_triple_node[fetch_value]):
                    dump_node(node_id_list=each_triple_node[fetch_value],
                        collection_name=node_collection)
                elif isinstance(each_triple_node[fetch_value], ObjectId):
                    dump_node(node_id=each_triple_node[fetch_value],
                            collection_name=node_collection)

class Command(BaseCommand):
    def handle(self, *args, **options):
        group_id = raw_input("Please enter group id:\t")
        group_node = node_collection.one({"_type":"Group","_id":ObjectId(group_id)})
        if group_node:
            print "\tRequest received for Export of : ", group_node.name
            nodes_falling_under_grp = node_collection.find({"group_set":ObjectId(group_node._id)})
            print "\n\tTotal GSystem objects found: ", nodes_falling_under_grp.count()
            confirm_export = raw_input("\n\tDo you want to continue? Enter y/n:\t ")
            if confirm_export == 'y' or confirm_export == 'Y':
                print "START : ", str(datetime.datetime.now())
                group_dump_path = setup_dump_path(group_node.name)
                create_factory_schema_mapper(group_dump_path)
                log_file_path = create_log_file(group_dump_path)

                print "*"*70
                print "\n Export will be found at: ", data_export_path
                print "\n Log will be found at: ", log_file_path

                print "\n This will take few minutes. Please be patient.\n"
                print "*"*70
                # import ipdb; ipdb.set_trace()

                call_group_export(group_node, nodes_falling_under_grp, num_of_processes=multiprocessing.cpu_count())
                get_counter_ids(group_id)
                # dump_media_data()
            global log_file
            log_file.write("\n*************************************************************")
            log_file.write("\n######### Script Completed at : " + str(datetime.datetime.now()) + " #########\n\n")
            print "END : ", str(datetime.datetime.now())

        else:
            print "\n Enter a valid ObjectId"

def call_group_export(group_node, nodes_cur, num_of_processes=4):
    '''
        Introducing multiprocessing to use cores available on the system to
        take dump of nodes of the entire group.
    '''
    nodes_cur = list(nodes_cur)
    # Include Group Object.
    nodes_cur.append(group_node)
    def worker(nodes_cur, out_q):
        for each_node in nodes_cur:
            print ".",
            dump_node(node=each_node,collection_name=node_collection)
            # node_collection_ids.add(each_node._id)
            if 'File' in each_node.member_of_names_list:
                get_file_node_details(each_node)

            if each_node.collection_set:
                get_nested_ids(each_node,'collection_set')

            if each_node.prior_node:
                get_nested_ids(each_node,'prior_node')

            if each_node.post_node:
                get_nested_ids(each_node,'post_node')

            #fetch triple_data
            get_triple_data(each_node._id)
    # Each process will get 'chunksize' student_cur and a queue to put his out
    # dict into
    out_q = multiprocessing.Queue()
    chunksize = int(math.ceil(len(nodes_cur) / float(num_of_processes)))
    procs = []

    for i in range(num_of_processes):
        p = multiprocessing.Process(
            target=worker,
            args=(nodes_cur[chunksize * i:chunksize * (i + 1)], out_q)
        )
        procs.append(p)
        p.start()

    # Collect all results into a single result list. We know how many lists
    # with results to expect.
    # resultlist = []
    # for i in range(num_of_processes):
    #     resultlist.extend(out_q.get())

    # Wait for all worker processes to finish
    for p in procs:
        p.join()

    # return resultlist

def build_rcs(node, collection_name):
    '''
    Updates the rcs json with the current node's strcuture that
    might have missed due to update queries.
    Runs a save() method on the node and calls copy_rcs()
    '''
    if node:
        global log_file
        try:
            if collection_name is triple_collection:
                if 'attribute_type' in node:
                    triple_node_RT_AT = node_collection.one({'_id': node.attribute_type})
                elif 'relation_type' in node:
                    triple_node_RT_AT = node_collection.one({'_id': node.relation_type})
                node.save(triple_node=triple_node_RT_AT, triple_id=triple_node_RT_AT._id)
            else:
                node.save()
            log_file.write("\n RCS Built for " + str(node._id) )
            copy_rcs(node)
        except Exception as buildRCSError:
            error_log = "\n !!! Error found while Building RCS ."
            error_log += "\nError: " + str(buildRCSError)
            log_file.write(error_log)
            print error_log
            pass

def copy_rcs(node):
    '''
    Actual copying of RCS files from /data/rcs-repo/ to export_path/rcs-repo
     of the nodes called from dump_node() and build_rcs()
    '''
    if node:
        global log_file
        try:
            # To update RCS
            path = historyMgr.get_file_path(node)
            path = path + ",v"

            if not os.path.exists(path):
                path = historyMgr.get_file_path(node)
                path = path + ",v"
                build_rcs.add(path)
            cp = "cp  -vu " + path + " " +" --parents " + data_export_path + "/"
            subprocess.Popen(cp,stderr=subprocess.STDOUT,shell=True)
            log_file.write("\n RCS Copied " + str(path) )

        except Exception as copyRCSError:
            error_log = "\n !!! Error found while Copying RCS ."
            error_log += "\nError: " + str(copyRCSError)
            log_file.write(error_log)
            print error_log
            pass

def dump_node(collection_name=node_collection, node=None, node_id=None, node_id_list=None):
    '''
    Receives all nodes pertaining to exporting group belonging to all existing collections.
    Calls build_rcs.
    '''
    try:
        global log_file
        log_file.write("\n dump_node invoked for: " + str(collection_name))
        if node:
            log_file.write("\tNode: " + str(node))
            build_rcs(node, collection_name)
            log_file.write("\n dump node finished for:  " + str(node._id) )
        elif node_id:
            log_file.write("\tNode_id : " + str(node_id))
            node = collection_name.one({'_id': ObjectId(node_id)})
            build_rcs(node, collection_name)
            log_file.write("\n dump node finished for:  " + str(node._id) )
        elif node_id_list:
            node_cur = collection_name.one({'_id': {'$in': node_id_list}})
            log_file.write("\tNode_id_list : " + str(node_id_list))
            for each_node in nodes_cur:
                build_rcs(node, collection_name)
                log_file.write("\n dump node finished for:  " + str(node._id) )

    except Exception as dump_err:
        error_log = "\n !!! Error found while taking dump in dump_node() ."
        error_log += "\nError: " + str(dump_err)
        log_file.write(error_log)
        print error_log
        pass

def dump_media_data(media_path):
    # Copy media file to /data/media location
    # print media_export_path
    global log_file
    log_file.write("\n--- Media Copying in process --- ", str(media_path))
    try:
        print "\n each_file_media_url: ", media_path
        fp = os.path.join(MEDIA_ROOT,media_path)
        if os.path.exists(fp):
            cp = "cp  -u " + fp + " " +" --parents " + media_export_path + "/"
            subprocess.Popen(cp,stderr=subprocess.STDOUT,shell=True)
            media_url_found.add(fp)
            log_file.write("\n Media Copied:  " + str(fp) )

        else:
            log_file.write("\n Media NOT Copied:  " + str(fp) )
            media_url_not_found.add(fp)
    except Exception as dumpMediaError:
        error_log = "\n !!! Error found while taking dump of Media.\n" +  str(media_path)
        error_log += "\nError: " + str(dumpMediaError)
        log_file.write(error_log)
        print error_log
        pass

def get_file_node_details(node):
    '''
    Check if_file field and take its dump
    'if_file': {
                    'mime_type': basestring,
                    'original': {'id': ObjectId, 'relurl': basestring},
                    'mid': {'id': ObjectId, 'relurl': basestring},
                    'thumbnail': {'id': ObjectId, 'relurl': basestring}
                },

    '''
    dump_node(node_id=node.if_file['original']['id'], collection_name=filehive_collection)
    dump_node(node_id=node.if_file['mid']['id'], collection_name=filehive_collection)
    dump_node(node_id=node.if_file['thumbnail']['id'], collection_name=filehive_collection)
    dump_media_data(node.if_file['original']['relurl'])
    dump_media_data(node.if_file['mid']['relurl'])
    dump_media_data(node.if_file['thumbnail']['relurl'])
    # if each_field == 'group_set':
    #     for each_grp_id in node.group_set:
    #         group_node = node_collection.find_one({"_id":ObjectId(each_grp_id)})
    #         if group_node and group_node._type != unicode('Author'):
    #             group_set.extend(group_node.group_set)
    # if each_field == 'author_set':
    #     user_list.extend(node.author_set)

def get_nested_ids(node,field_name):
    '''
        Recursive function to fetch Objectids from a
        particular field of passed node.
        field_name can be : collection_set, post_node, prior_node
    '''
    if node[field_name]:
        for each_id in node[field_name]:
            each_node = node_collection.one({"_id":ObjectId(each_id)})
            dump_node(node=each_node, collection_name=node_collection)
            if each_node and each_node[field_name]:
                get_nested_ids(each_node, field_name)

def get_counter_ids(group_id):
    '''
    Fetch all the Counter instances of the exporting Group
    '''
    counter_collection_cur = counter_collection.find({'group_id':ObjectId(group_id)})
    if counter_collection_cur :
        for each_obj in counter_collection_cur :
            dump_node(node=each_obj,collection_name=counter_collection)
<commit_msg>Export: 1. prompt for fork or clone qtn & 2. creation of configs file<commit_after>import os
import datetime
import subprocess
import multiprocessing
import math
try:
    from bson import ObjectId
except ImportError:  # old pymongo
    from pymongo.objectid import ObjectId
from django.core.management.base import BaseCommand, CommandError
from gnowsys_ndf.ndf.models import node_collection, triple_collection, filehive_collection, counter_collection
from gnowsys_ndf.ndf.models import HistoryManager
from gnowsys_ndf.settings import GSTUDIO_DATA_ROOT, GSTUDIO_LOGS_DIR_PATH, MEDIA_ROOT, GSTUDIO_INSTITUTE_ID
from schema_mapping import create_factory_schema_mapper

node_collection_ids = set()
triple_collection_ids = set()
filehives_collection_ids = set()
counter_collection_ids = set()
filehives_media_urls = set()
build_rcs = set()
rcs_paths_found = set()
media_url_not_found = set()
media_url_found = set()
historyMgr = HistoryManager()
data_export_path = None
media_export_path = None
log_file = None
IS_FORK = False
IS_CLONE = False
RESTORE_USER_DATA = False

def create_log_file(dump_path):
    '''
        Creates log file in gstudio-logs/ with
        the name of the dump folder
    '''
    dump_path = dump_path.split("/")[-1]
    log_file_name = 'group_dump_' + str(dump_path)+ '.log'
    if not os.path.exists(GSTUDIO_LOGS_DIR_PATH):
        os.makedirs(GSTUDIO_LOGS_DIR_PATH)

    log_file_path = os.path.join(GSTUDIO_LOGS_DIR_PATH, log_file_name)
    # print log_file_path
    global log_file
    log_file = open(log_file_path, 'w+')
    log_file.write("\n######### Script ran on : " + str(datetime.datetime.now()) + " #########\n\n")
    return log_file_path

def setup_dump_path(group_name):
    '''
        Creates factory_schema.json which will hold basic info
        like ObjectId, name, type of TYPES_LIST and GSTUDIO_DEFAULT_GROUPS
    '''
    global data_export_path
    global media_export_path
    datetimestamp = datetime.datetime.now().isoformat()
    data_export_path = os.path.join(GSTUDIO_DATA_ROOT, 'data_export', group_name + "_" + str(datetimestamp))
    media_export_path = os.path.join(data_export_path, 'media_files')
    if not os.path.exists(data_export_path):
        os.makedirs(data_export_path)
    if not os.path.exists(media_export_path):
        os.makedirs(media_export_path)
    return data_export_path

def create_configs_file(group_id, data_dump_path):
    global IS_FORK
    global IS_CLONE
    global RESTORE_USER_DATA
    configs_file_path = os.path.join(data_dump_path, "configs.py")
    with open(configs_file_path, 'w+') as configs_file_out:
        configs_file_out.write("\nFORK=" + str(IS_FORK))
        configs_file_out.write("\nCLONE=" + str(IS_CLONE))
        configs_file_out.write("\nRESTORE_USER_DATA=" + str(RESTORE_USER_DATA))
        configs_file_out.write("\nGSTUDIO_INSTITUTE_ID=" + str(GSTUDIO_INSTITUTE_ID))
        configs_file_out.write("\nGROUP_ID=" + str(group_id))

def get_triple_data(node_id):
    '''
    Gets all data stored in triples for this node.
    Fetches GAttrtibutes as wells as GRelations.
    '''
    triple_query = {"_type": {'$in': ["GAttribute", "GRelation"]}, "subject": node_id}

    node_gattr_grel_cur = triple_collection.find(triple_query)
    fetch_value = None
    if node_gattr_grel_cur:

        for each_triple_node in node_gattr_grel_cur:
            dump_node(node=each_triple_node,
                collection_name=triple_collection)
            # Get ObjectIds in object_value fields
            if each_triple_node._type is "GAttribute":
                fetch_value = "object_value"
            elif each_triple_node._type is "GRelation":
                fetch_value = "right_subject"
            if fetch_value:
                print "\n fetch_value -- ", fetch_value
                if type(each_triple_node[fetch_value]) == list and all(isinstance(each_obj_value, ObjectId) for each_obj_value in each_triple_node[fetch_value]):
                    dump_node(node_id_list=each_triple_node[fetch_value],
                        collection_name=node_collection)
                elif isinstance(each_triple_node[fetch_value], ObjectId):
                    dump_node(node_id=each_triple_node[fetch_value],
                            collection_name=node_collection)

class Command(BaseCommand):
    def handle(self, *args, **options):
        group_id = raw_input("\n\tPlease enter group id:\t")
        try:

            group_node = node_collection.one({"_type":"Group","_id":ObjectId(group_id)})
        except Exception as group_id_error:
            print "\n Enter a valid ObjectId"
            print "\n Exiting..."
            os._exit(0)

        if group_node:
            print "\tRequest received for Export of : ", group_node.name
            fork_clone_opt = "\n\tExport Options:"
            fork_clone_opt += "\n\t\t 1. Fork (What is Fork?"
            fork_clone_opt += " Restore will create 'New' instances(new Ids)"
            fork_clone_opt += "\n\t\t 2. Clone (What is Clone?"
            fork_clone_opt += " Restore will NOT create 'New' instances(new Ids)"
            fork_clone_opt += "\n\tEnter options 1 or 2 or any other key to cancel: \t"

            fork_clone_confirm = raw_input(fork_clone_opt)
            global IS_FORK
            global IS_CLONE
            global RESTORE_USER_DATA
            if fork_clone_confirm == '1':
                print "\n\t!!! Chosen FORK option !!!"
                IS_FORK = True
                IS_CLONE = False
            elif fork_clone_confirm == '2':
                print "\n\t!!! Chosen CLONE option !!!"
                IS_FORK = False
                IS_CLONE = True
            else:
                print "\n Exiting..."
                os._exit(0)
            user_data_dump = raw_input("\n\tDo you want to include Users in this export ? Enter y/n:\t ")
            if user_data_dump == 'y' or user_data_dump == 'Y':
                RESTORE_USER_DATA = True
            else:
                RESTORE_USER_DATA = False

            nodes_falling_under_grp = node_collection.find({"group_set":ObjectId(group_node._id)})
            print "\n\tTotal objects found: ", nodes_falling_under_grp.count()
            confirm_export = raw_input("\n\tDo you want to continue? Enter y/n:\t ")
            if confirm_export is 'y' or confirm_export is 'Y':
                print "START : ", str(datetime.datetime.now())
                group_dump_path = setup_dump_path(group_node.name)
                create_factory_schema_mapper(group_dump_path)
                create_configs_file(group_node._id, group_dump_path)
                log_file_path = create_log_file(group_dump_path)

                print "*"*70
                print "\n Export will be found at: ", data_export_path
                print "\n Log will be found at: ", log_file_path

                print "\n This will take few minutes. Please be patient.\n"
                print "*"*70
                # import ipdb; ipdb.set_trace()

                call_group_export(group_node, nodes_falling_under_grp, num_of_processes=multiprocessing.cpu_count())
                get_counter_ids(group_id)
                # dump_media_data()
                global log_file
                log_file.write("\n*************************************************************")
                log_file.write("\n######### Script Completed at : " + str(datetime.datetime.now()) + " #########\n\n")
                print "END : ", str(datetime.datetime.now())
            else:
                print "\n Exiting..."
                os._exit(0)

        else:
            print "\n Enter a valid ObjectId"
            print "\n Exiting..."
            os._exit(0)

def call_group_export(group_node, nodes_cur, num_of_processes=4):
    '''
        Introducing multiprocessing to use cores available on the system to
        take dump of nodes of the entire group.
    '''
    nodes_cur = list(nodes_cur)
    # Include Group Object.
    nodes_cur.append(group_node)
    def worker(nodes_cur, out_q):
        for each_node in nodes_cur:
            print ".",
            dump_node(node=each_node,collection_name=node_collection)
            # node_collection_ids.add(each_node._id)
            if 'File' in each_node.member_of_names_list:
                get_file_node_details(each_node)

            if each_node.collection_set:
                get_nested_ids(each_node,'collection_set')

            if each_node.prior_node:
                get_nested_ids(each_node,'prior_node')

            if each_node.post_node:
                get_nested_ids(each_node,'post_node')

            #fetch triple_data
            get_triple_data(each_node._id)
    # Each process will get 'chunksize' student_cur and a queue to put his out
    # dict into
    out_q = multiprocessing.Queue()
    chunksize = int(math.ceil(len(nodes_cur) / float(num_of_processes)))
    procs = []

    for i in range(num_of_processes):
        p = multiprocessing.Process(
            target=worker,
            args=(nodes_cur[chunksize * i:chunksize * (i + 1)], out_q)
        )
        procs.append(p)
        p.start()

    # Collect all results into a single result list. We know how many lists
    # with results to expect.
    # resultlist = []
    # for i in range(num_of_processes):
    #     resultlist.extend(out_q.get())

    # Wait for all worker processes to finish
    for p in procs:
        p.join()

    # return resultlist

def build_rcs(node, collection_name):
    '''
    Updates the rcs json with the current node's strcuture that
    might have missed due to update queries.
    Runs a save() method on the node and calls copy_rcs()
    '''
    if node:
        global log_file
        try:
            if collection_name is triple_collection:
                if 'attribute_type' in node:
                    triple_node_RT_AT = node_collection.one({'_id': node.attribute_type})
                elif 'relation_type' in node:
                    triple_node_RT_AT = node_collection.one({'_id': node.relation_type})
                node.save(triple_node=triple_node_RT_AT, triple_id=triple_node_RT_AT._id)
            else:
                node.save()
            log_file.write("\n RCS Built for " + str(node._id) )
            copy_rcs(node)
        except Exception as buildRCSError:
            error_log = "\n !!! Error found while Building RCS ."
            error_log += "\nError: " + str(buildRCSError)
            log_file.write(error_log)
            print error_log
            pass

def copy_rcs(node):
    '''
    Actual copying of RCS files from /data/rcs-repo/ to export_path/rcs-repo
     of the nodes called from dump_node() and build_rcs()
    '''
    if node:
        global log_file
        try:
            # To update RCS
            path = historyMgr.get_file_path(node)
            path = path + ",v"

            if not os.path.exists(path):
                path = historyMgr.get_file_path(node)
                path = path + ",v"
                build_rcs.add(path)

            cp = "cp  -vu " + path + " " +" --parents " + data_export_path + "/"
            subprocess.Popen(cp,stderr=subprocess.STDOUT,shell=True)

            log_file.write("\n RCS Copied " + str(path) )

        except Exception as copyRCSError:
            error_log = "\n !!! Error found while Copying RCS ."
            error_log += "\nError: " + str(copyRCSError)
            log_file.write(error_log)
            print error_log
            pass

def dump_node(collection_name=node_collection, node=None, node_id=None, node_id_list=None):
    '''
    Receives all nodes pertaining to exporting group belonging to all existing collections.
    Calls build_rcs.
    '''
    try:
        global log_file
        log_file.write("\n dump_node invoked for: " + str(collection_name))
        if node:
            log_file.write("\tNode: " + str(node))
            build_rcs(node, collection_name)
            log_file.write("\n dump node finished for:  " + str(node._id) )
        elif node_id:
            log_file.write("\tNode_id : " + str(node_id))
            node = collection_name.one({'_id': ObjectId(node_id)})
            build_rcs(node, collection_name)
            log_file.write("\n dump node finished for:  " + str(node._id) )
        elif node_id_list:
            node_cur = collection_name.one({'_id': {'$in': node_id_list}})
            log_file.write("\tNode_id_list : " + str(node_id_list))
            for each_node in nodes_cur:
                build_rcs(node, collection_name)
                log_file.write("\n dump node finished for:  " + str(node._id) )

    except Exception as dump_err:
        error_log = "\n !!! Error found while taking dump in dump_node() ."
        error_log += "\nError: " + str(dump_err)
        log_file.write(error_log)
        print error_log
        pass

def dump_media_data(media_path):
    # Copy media file to /data/media location
    # print media_export_path
    global log_file
    log_file.write("\n--- Media Copying in process --- ", str(media_path))
    try:
        print "\n each_file_media_url: ", media_path
        fp = os.path.join(MEDIA_ROOT,media_path)
        if os.path.exists(fp):
            cp = "cp  -u " + fp + " " +" --parents " + media_export_path + "/"
            subprocess.Popen(cp,stderr=subprocess.STDOUT,shell=True)
            media_url_found.add(fp)
            log_file.write("\n Media Copied:  " + str(fp) )

        else:
            log_file.write("\n Media NOT Copied:  " + str(fp) )
            media_url_not_found.add(fp)
    except Exception as dumpMediaError:
        error_log = "\n !!! Error found while taking dump of Media.\n" +  str(media_path)
        error_log += "\nError: " + str(dumpMediaError)
        log_file.write(error_log)
        print error_log
        pass

def get_file_node_details(node):
    '''
    Check if_file field and take its dump
    'if_file': {
                    'mime_type': basestring,
                    'original': {'id': ObjectId, 'relurl': basestring},
                    'mid': {'id': ObjectId, 'relurl': basestring},
                    'thumbnail': {'id': ObjectId, 'relurl': basestring}
                },

    '''
    dump_node(node_id=node.if_file['original']['id'], collection_name=filehive_collection)
    dump_node(node_id=node.if_file['mid']['id'], collection_name=filehive_collection)
    dump_node(node_id=node.if_file['thumbnail']['id'], collection_name=filehive_collection)
    dump_media_data(node.if_file['original']['relurl'])
    dump_media_data(node.if_file['mid']['relurl'])
    dump_media_data(node.if_file['thumbnail']['relurl'])
    # if each_field == 'group_set':
    #     for each_grp_id in node.group_set:
    #         group_node = node_collection.find_one({"_id":ObjectId(each_grp_id)})
    #         if group_node and group_node._type != unicode('Author'):
    #             group_set.extend(group_node.group_set)
    # if each_field == 'author_set':
    #     user_list.extend(node.author_set)

def get_nested_ids(node,field_name):
    '''
        Recursive function to fetch Objectids from a
        particular field of passed node.
        field_name can be : collection_set, post_node, prior_node
    '''
    if node[field_name]:
        for each_id in node[field_name]:
            each_node = node_collection.one({"_id":ObjectId(each_id)})
            dump_node(node=each_node, collection_name=node_collection)
            if each_node and each_node[field_name]:
                get_nested_ids(each_node, field_name)

def get_counter_ids(group_id):
    '''
    Fetch all the Counter instances of the exporting Group
    '''
    counter_collection_cur = counter_collection.find({'group_id':ObjectId(group_id)})
    if counter_collection_cur :
        for each_obj in counter_collection_cur :
            dump_node(node=each_obj,collection_name=counter_collection)
<|endoftext|>