<commit_before>#!/usr/bin/python
"""
based on: "Tutorial: Making your own Hex Dump Program" by DrapsTV
https://www.youtube.com/watch?v=B8nRrw_M_nk&index=1&list=WL

"""
import argparse
import os
import itertools
import binascii

ASCII = 'ascii'
CTRL = 'ctrl'
OTHER = 'other'

# https://en.wikipedia.org/wiki/List_of_file_signatures
FILE_SIGNATURES = {
    'a1b2c3d4': 'Libpcap File Format',
    'd4c3b2a1': 'Libpcap File Format',
    '0a0d0d0a': 'PCAP Next Generation Dump File Format (pcapng)',
    'edabeedb': 'RedHat Package Manager (RPM) package',
    '53503031': 'Amazon Kindle Update Package',
    '00': 'IBM Storyboard bitmap file; Windows Program Information File; Mac Stuffit Self-Extracting Archive; IRIS OCR data file',
    'BEBAFECA': 'Palm Desktop Calendar Archive',
    '00014244': 'Palm Desktop To Do Archive',
    '00014454': 'Palm Desktop Calendar Archive',
    '00010000': 'Palm Desktop Data File (Access format)',
    '00000100': 'Computer icon encoded in ICO file format',
    '667479703367': '3rd Generation Partnership Project 3GPP and 3GPP2 multimedia files',
    '1FA0': 'Compressed file (often tar zip) using LZH algorithm',
    '425A68': 'Compressed file using Bzip2 algorithm',
    '474946383761': 'Image file encoded in the Graphics Interchange Format (GIF) - GIF87a',
    '474946383961': 'Image file encoded in the Graphics Interchange Format (GIF) - GIF89a',
    'FFD8FFE00010': 'JPEG raw or in the JFIF or Exif file format',
    '49492A00': 'Tagged Image File Format (tiff, Little Endian format)',
    '4D4D002A': 'Tagged Image File Format (tiff, Big Endian)',
    '49492A0010000000': 'Canon RAW Format Version 2',
    '4352':'Canon RAW format is based on the TIFF file format',
    '802A5FD7': 'Kodak Cineon image',
    '524E4301':'Compressed file using Rob Northen Compression version 1',
    '524E4302':'Compressed file using Rob Northen Compression version 2',
    '53445058':'SMPTE DPX image (big endian format)',
    '58504453':'SMPTE DPX image (little endian format)',
    '762F3101': 'OpenEXR image',
    '425047FB':'Better Portable Graphics',
    'FFD8FFDB':'JPEG raw or in the JFIF or Exif file format',
    'FFD8FFE000104A4649460001':'JPEG raw or in the JFIF or Exif file format',
    'FFD8FFE1':'JPEG raw or in the JFIF or Exif file format',
    '464F52':'IFF Interleaved Bitmap Image',
    '4C424D':'IFF Interleaved Bitmap Image',
    '464F52':'IFF 8-Bit Sampled Voice',
    '535658':'IFF 8-Bit Sampled Voice',
    '464F524D': 'IFF (misc filetypes)',
    '4C5A4950': 'lzip compressed file',
    '4D5A':'DOS MZ executable file format and its descendants (including NE and PE)',
    '504B0304':'zip file format and formats based on it, such as JAR, ODF, OOXML',
    '504B0506':'zip file format and formats based on it, such as JAR, ODF, OOXML (empty archive)',
    '504B0708':'zip file format and formats based on it, such as JAR, ODF, OOXML (spanned archive)',
    '526172211A0700':'RAR archive version 1.50 onwards',
    '526172211A070100':'RAR archive version 5.0 onwards',
    '7F454C46': 'ELF Executable and Linkable Format'

}

COLORS = {
    "black": '\33[0;30m\33[40m',
    "white": '\33[0;37m\33[40m',
    "red": '\33[0;31m\33[40m',
    "green": '\33[0;32m\33[40m',
    "green_bg": '\33[1;32m\33[41m',
    "yellow": '\33[0;33m\33[40m',
    "yellow_bg": '\33[1;33m\33[41m',
    "blue": '\33[0;34m\33[40m',
    "magenta": '\33[0;35m\33[40m',
    "magenta_bg": '\33[1;35m\33[41m',
    "cyan": '\33[0;36m\33[40m',
    "grey": '\33[0;90m\33[40m',
    "lightgrey": '\33[0;37m\33[40m',
    "lightblue": '\33[0;94m\33[40m'
}

def file_type(file_signature):
    """
    Recognize file type based on file 'magic numbers'
    """
    file_signature = binascii.hexlify(file_signature).upper()
    recognized = 'ASCII text (no file signature found)'
    for signature in FILE_SIGNATURES:
        if file_signature.startswith(signature.upper()):
            recognized = FILE_SIGNATURES[signature]

    return "{}\n[+] File type: {}{}{}".format(COLORS['cyan'], COLORS['yellow'], recognized, COLORS['white'])

def char_type(c):
    """
    Returns char type depends on its ASCII code
    """
    if 32 < ord(c) < 128:
        return ASCII
    if ord(c) <= 16:
        return CTRL
    return OTHER

def make_color(c, df_c=False):
    """
    Formats color for byte depends on if it's printable ASCII
    """

    # for file diff - if characters are different, use bg color for char:
    diff = (c != df_c) if df_c != False else False
    # printable ASCII:
    if char_type(c) == ASCII:
        if diff:
            retval = "{}{:02X}{}".format(
                COLORS['green_bg'], ord(c), COLORS['white'])
        else:
            retval = "{}{:02X}{}".format(
                COLORS['green'], ord(c), COLORS['white'])
    if char_type(c) == OTHER:
        if diff:
            retval = "{}{:02X}{}".format(
                COLORS['yellow_bg'], ord(c), COLORS['white'])
        else:
            retval = "{}{:02X}{}".format(
                COLORS['yellow'], ord(c), COLORS['white'])
    if char_type(c) == CTRL:
        if diff:
            retval = "{}{:02X}{}".format(
                COLORS['magenta_bg'], ord(c), COLORS['white'])
        else:
            retval = "{}{:02X}{}".format(
                COLORS['magenta'], ord(c), COLORS['white'])

    return retval

def format_text(c):
    """
    Formats color for character depends on if it's printable ASCII
    """
    if char_type(c) == ASCII:
        retval = "{}{}{}".format(COLORS['lightblue'], c, COLORS['white'])
    if char_type(c) == CTRL:
        retval = "{}.{}".format(COLORS['magenta'], COLORS['white'])
    if char_type(c) == OTHER:
        retval = "{}.{}".format(COLORS['yellow'], COLORS['white'])
    return retval

def format_chunk(chunk, start, stop, df_chunk=False, dec=False):
    """
    Formats one full chunk (byte)
    """
    if dec:
        if df_chunk:
            return " ".join("{}:{}{:#04}{} ".format(make_color(c, df_c), COLORS['grey'],
                                                    ord(c), COLORS['white']) for c, df_c in itertools.izip(chunk[start:stop], df_chunk[start:stop]))
        return " ".join("{}:{}{:#04}{} ".format(make_color(c), COLORS['grey'],
                                                ord(c), COLORS['white']) for c in chunk[start:stop])
    else:
        if df_chunk:
            return " ".join("{} ".format(make_color(c, df_c))
                            for c, df_c in itertools.izip(chunk[start:stop], df_chunk[start:stop]))
        return " ".join("{} ".format(make_color(c)) for c in chunk[start:stop])

def extract_shellcode(start, end, read_binary):
    """
    Extract shellcode in hex format, eg. for C exploits, from given range of bytes
    """
    read_binary.seek(start)
    shellcode = ""
    s = read_binary.read(end - start)
    for c in s:
        if ord(c) == 0:
            shellcode = shellcode + "{}".format(COLORS['red']) + str(
                hex(ord(c))).replace("0x", "\\x") + "{}".format(COLORS['white'])
        else:
            shellcode = shellcode + "{}".format(COLORS['yellow']) + str(
                hex(ord(c))).replace("0x", "\\x") + "{}".format(COLORS['white'])
    print "\n{}[+] Shellcode extracted from byte(s) {:#08x} to {:#08x}:{}".format(COLORS['cyan'], start, end, COLORS['white'])
    print "\n{}\n".format(shellcode)

if __name__ == "__main__":
    """
    main program routine
    """
    __FROM = 0
    __TO = 0

    parser = argparse.ArgumentParser()
    parser.add_argument("file", help="Specify a file")
    parser.add_argument(
        "-d", "--decimal", help="Display DEC values with HEX", action="store_true")
    parser.add_argument(
        "-s", "--start", help="Start byte")
    parser.add_argument(
        "-e", "--end", help="End byte")
    parser.add_argument(
        "-D", "--diff", help="Perform diff with FILENAME")
    parser.add_argument(
        "-S", "--shellcode", help="Extract shellcode (-s and -e has to be passed)", action="store_true")

    args = parser.parse_args()
    b = 16

    # for -D / --diff - second file has to be opened
    # https://github.com/bl4de/security-tools/issues/22 [RESOLVED]
    if args.diff:
        diff_file = open(args.diff, 'rb')

    if args.file:
        # read first 8 bytes to recognize file type
        print file_type(open(args.file, 'rb').read(8))

        with open(args.file, 'rb') as infile:
            if args.start > -1 and args.end and (int(args.start, 16) > -1 and int(args.end, 16) > int(args.start, 16)):
                __FROM = int(args.start, 16)
                __TO = int(args.end, 16)
            else:
                __TO = os.path.getsize(args.file)

            if args.shellcode and __FROM > -1 and __TO:
                extract_shellcode(__FROM, __TO, infile)

            infile.seek(__FROM)

            if args.diff:
                diff_file.seek(__FROM)

            offset = __FROM

            print "{}[+] Hex dump: {}\n".format(COLORS['cyan'], COLORS['white'])
            while offset < __TO:
                chunk = infile.read(b)

                if args.diff:
                    df_chunk = diff_file.read(b)
                else:
                    df_chunk = False

                if len(chunk) == 0:
                    break

                text = str(chunk)
                text = ''.join([format_text(i) for i in text])

                output = "{}{:#08x}{}".format(
                    COLORS['cyan'], offset, COLORS['white']) + ": "

                output += format_chunk(chunk, 0, 4,
                                       df_chunk, args.decimal) + " | "
                output += format_chunk(chunk, 4, 8,
                                       df_chunk, args.decimal) + " | "
                output += format_chunk(chunk, 8, 12,
                                       df_chunk, args.decimal) + " | "
                output += format_chunk(chunk, 12, 16, df_chunk, args.decimal)

                if args.diff:
                    df_text = str(df_chunk)
                    df_text = ''.join([format_text(i) for i in df_text])

                    df_output = "    " + \
                        format_chunk(df_chunk, 0, 4, chunk,
                                     args.decimal) + " | "
                    df_output += format_chunk(df_chunk,
                                              4, 8, chunk, args.decimal) + " | "
                    df_output += format_chunk(df_chunk,
                                              8, 12, chunk, args.decimal) + " | "
                    df_output += format_chunk(df_chunk, 12,
                                              16, chunk, args.decimal) + df_text

                if len(chunk) % b != 0:
                    if args.decimal:
                        output += "   " * (((b * 2) - 4 - len(chunk))) + text
                        if args.diff:
                            df_output += "   " * \
                                (((b * 2) - 4 - len(df_chunk))) + df_text
                    else:
                        output += "   " * (b + 4 - len(chunk)) + text
                        if args.diff:
                            df_output += "   " * \
                                (b + 4 - len(df_chunk)) + df_text
                else:
                    output += " " + text
                    if args.diff:
                        output += df_output

                print output
                offset += 16

            print
    else:
        print parser.usage
<commit_msg>[hexview] file types c.d.<commit_after>#!/usr/bin/python
"""
based on: "Tutorial: Making your own Hex Dump Program" by DrapsTV
https://www.youtube.com/watch?v=B8nRrw_M_nk&index=1&list=WL

"""
import argparse
import os
import itertools
import binascii

ASCII = 'ascii'
CTRL = 'ctrl'
OTHER = 'other'

# https://en.wikipedia.org/wiki/List_of_file_signatures
FILE_SIGNATURES = {
    'a1b2c3d4': 'Libpcap File Format',
    'd4c3b2a1': 'Libpcap File Format',
    '0a0d0d0a': 'PCAP Next Generation Dump File Format (pcapng)',
    'edabeedb': 'RedHat Package Manager (RPM) package',
    '53503031': 'Amazon Kindle Update Package',
    '00': 'IBM Storyboard bitmap file; Windows Program Information File; Mac Stuffit Self-Extracting Archive; IRIS OCR data file',
    'BEBAFECA': 'Palm Desktop Calendar Archive',
    '00014244': 'Palm Desktop To Do Archive',
    '00014454': 'Palm Desktop Calendar Archive',
    '00010000': 'Palm Desktop Data File (Access format)',
    '00000100': 'Computer icon encoded in ICO file format',
    '667479703367': '3rd Generation Partnership Project 3GPP and 3GPP2 multimedia files',
    '1FA0': 'Compressed file (often tar zip) using LZH algorithm',
    '425A68': 'Compressed file using Bzip2 algorithm',
    '474946383761': 'Image file encoded in the Graphics Interchange Format (GIF) - GIF87a',
    '474946383961': 'Image file encoded in the Graphics Interchange Format (GIF) - GIF89a',
    'FFD8FFE00010': 'JPEG raw or in the JFIF or Exif file format',
    '49492A00': 'Tagged Image File Format (tiff, Little Endian format)',
    '4D4D002A': 'Tagged Image File Format (tiff, Big Endian)',
    '49492A0010000000': 'Canon RAW Format Version 2',
    '4352':'Canon RAW format is based on the TIFF file format',
    '802A5FD7': 'Kodak Cineon image',
    '524E4301':'Compressed file using Rob Northen Compression version 1',
    '524E4302':'Compressed file using Rob Northen Compression version 2',
    '53445058':'SMPTE DPX image (big endian format)',
    '58504453':'SMPTE DPX image (little endian format)',
    '762F3101': 'OpenEXR image',
    '425047FB':'Better Portable Graphics',
    'FFD8FFDB':'JPEG raw or in the JFIF or Exif file format',
    'FFD8FFE000104A4649460001':'JPEG raw or in the JFIF or Exif file format',
    'FFD8FFE1':'JPEG raw or in the JFIF or Exif file format',
    '464F52':'IFF Interleaved Bitmap Image',
    '4C424D':'IFF Interleaved Bitmap Image',
    '464F52':'IFF 8-Bit Sampled Voice',
    '535658':'IFF 8-Bit Sampled Voice',
    '464F524D': 'IFF (misc filetypes)',
    '4C5A4950': 'lzip compressed file',
    '4D5A':'DOS MZ executable file format and its descendants (including NE and PE)',
    '504B0304':'zip file format and formats based on it, such as JAR, ODF, OOXML',
    '504B0506':'zip file format and formats based on it, such as JAR, ODF, OOXML (empty archive)',
    '504B0708':'zip file format and formats based on it, such as JAR, ODF, OOXML (spanned archive)',
    '526172211A0700':'RAR archive version 1.50 onwards',
    '526172211A070100':'RAR archive version 5.0 onwards',
    '7F454C46': 'ELF Executable and Linkable Format',
    '89504E470D0A1A0A':'Image encoded in the Portable Network Graphics format (PNG)',
    'CAFEBABE':'Java class file, Mach-O Fat Binary',
    'EFBBBF':'UTF-8 encoded Unicode byte order mark, commonly seen in text files'

}

COLORS = {
    "black": '\33[0;30m\33[40m',
    "white": '\33[0;37m\33[40m',
    "red": '\33[0;31m\33[40m',
    "green": '\33[0;32m\33[40m',
    "green_bg": '\33[1;32m\33[41m',
    "yellow": '\33[0;33m\33[40m',
    "yellow_bg": '\33[1;33m\33[41m',
    "blue": '\33[0;34m\33[40m',
    "magenta": '\33[0;35m\33[40m',
    "magenta_bg": '\33[1;35m\33[41m',
    "cyan": '\33[0;36m\33[40m',
    "grey": '\33[0;90m\33[40m',
    "lightgrey": '\33[0;37m\33[40m',
    "lightblue": '\33[0;94m\33[40m'
}

def file_type(file_signature):
    """
    Recognize file type based on file 'magic numbers'
    """
    file_signature = binascii.hexlify(file_signature).upper()
    recognized = 'ASCII text (no file signature found)'
    for signature in FILE_SIGNATURES:
        if file_signature.startswith(signature.upper()):
            recognized = FILE_SIGNATURES[signature]

    return "{}\n[+] File type: {}{}{}".format(COLORS['cyan'], COLORS['yellow'], recognized, COLORS['white'])

def char_type(c):
    """
    Returns char type depends on its ASCII code
    """
    if 32 < ord(c) < 128:
        return ASCII
    if ord(c) <= 16:
        return CTRL
    return OTHER

def make_color(c, df_c=False):
    """
    Formats color for byte depends on if it's printable ASCII
    """

    # for file diff - if characters are different, use bg color for char:
    diff = (c != df_c) if df_c != False else False
    # printable ASCII:
    if char_type(c) == ASCII:
        if diff:
            retval = "{}{:02X}{}".format(
                COLORS['green_bg'], ord(c), COLORS['white'])
        else:
            retval = "{}{:02X}{}".format(
                COLORS['green'], ord(c), COLORS['white'])
    if char_type(c) == OTHER:
        if diff:
            retval = "{}{:02X}{}".format(
                COLORS['yellow_bg'], ord(c), COLORS['white'])
        else:
            retval = "{}{:02X}{}".format(
                COLORS['yellow'], ord(c), COLORS['white'])
    if char_type(c) == CTRL:
        if diff:
            retval = "{}{:02X}{}".format(
                COLORS['magenta_bg'], ord(c), COLORS['white'])
        else:
            retval = "{}{:02X}{}".format(
                COLORS['magenta'], ord(c), COLORS['white'])

    return retval

def format_text(c):
    """
    Formats color for character depends on if it's printable ASCII
    """
    if char_type(c) == ASCII:
        retval = "{}{}{}".format(COLORS['lightblue'], c, COLORS['white'])
    if char_type(c) == CTRL:
        retval = "{}.{}".format(COLORS['magenta'], COLORS['white'])
    if char_type(c) == OTHER:
        retval = "{}.{}".format(COLORS['yellow'], COLORS['white'])
    return retval

def format_chunk(chunk, start, stop, df_chunk=False, dec=False):
    """
    Formats one full chunk (byte)
    """
    if dec:
        if df_chunk:
            return " ".join("{}:{}{:#04}{} ".format(make_color(c, df_c), COLORS['grey'],
                                                    ord(c), COLORS['white']) for c, df_c in itertools.izip(chunk[start:stop], df_chunk[start:stop]))
        return " ".join("{}:{}{:#04}{} ".format(make_color(c), COLORS['grey'],
                                                ord(c), COLORS['white']) for c in chunk[start:stop])
    else:
        if df_chunk:
            return " ".join("{} ".format(make_color(c, df_c))
                            for c, df_c in itertools.izip(chunk[start:stop], df_chunk[start:stop]))
        return " ".join("{} ".format(make_color(c)) for c in chunk[start:stop])

def extract_shellcode(start, end, read_binary):
    """
    Extract shellcode in hex format, eg. for C exploits, from given range of bytes
    """
    read_binary.seek(start)
    shellcode = ""
    s = read_binary.read(end - start)
    for c in s:
        if ord(c) == 0:
            shellcode = shellcode + "{}".format(COLORS['red']) + str(
                hex(ord(c))).replace("0x", "\\x") + "{}".format(COLORS['white'])
        else:
            shellcode = shellcode + "{}".format(COLORS['yellow']) + str(
                hex(ord(c))).replace("0x", "\\x") + "{}".format(COLORS['white'])
    print "\n{}[+] Shellcode extracted from byte(s) {:#08x} to {:#08x}:{}".format(COLORS['cyan'], start, end, COLORS['white'])
    print "\n{}\n".format(shellcode)

if __name__ == "__main__":
    """
    main program routine
    """
    __FROM = 0
    __TO = 0

    parser = argparse.ArgumentParser()
    parser.add_argument("file", help="Specify a file")
    parser.add_argument(
        "-d", "--decimal", help="Display DEC values with HEX", action="store_true")
    parser.add_argument(
        "-s", "--start", help="Start byte")
    parser.add_argument(
        "-e", "--end", help="End byte")
    parser.add_argument(
        "-D", "--diff", help="Perform diff with FILENAME")
    parser.add_argument(
        "-S", "--shellcode", help="Extract shellcode (-s and -e has to be passed)", action="store_true")

    args = parser.parse_args()
    b = 16

    # for -D / --diff - second file has to be opened
    # https://github.com/bl4de/security-tools/issues/22 [RESOLVED]
    if args.diff:
        diff_file = open(args.diff, 'rb')

    if args.file:
        # read first 8 bytes to recognize file type
        print file_type(open(args.file, 'rb').read(8))

        with open(args.file, 'rb') as infile:
            if args.start > -1 and args.end and (int(args.start, 16) > -1 and int(args.end, 16) > int(args.start, 16)):
                __FROM = int(args.start, 16)
                __TO = int(args.end, 16)
            else:
                __TO = os.path.getsize(args.file)

            if args.shellcode and __FROM > -1 and __TO:
                extract_shellcode(__FROM, __TO, infile)

            infile.seek(__FROM)

            if args.diff:
                diff_file.seek(__FROM)

            offset = __FROM

            print "{}[+] Hex dump: {}\n".format(COLORS['cyan'], COLORS['white'])
            while offset < __TO:
                chunk = infile.read(b)

                if args.diff:
                    df_chunk = diff_file.read(b)
                else:
                    df_chunk = False

                if len(chunk) == 0:
                    break

                text = str(chunk)
                text = ''.join([format_text(i) for i in text])

                output = "{}{:#08x}{}".format(
                    COLORS['cyan'], offset, COLORS['white']) + ": "

                output += format_chunk(chunk, 0, 4,
                                       df_chunk, args.decimal) + " | "
                output += format_chunk(chunk, 4, 8,
                                       df_chunk, args.decimal) + " | "
                output += format_chunk(chunk, 8, 12,
                                       df_chunk, args.decimal) + " | "
                output += format_chunk(chunk, 12, 16, df_chunk, args.decimal)

                if args.diff:
                    df_text = str(df_chunk)
                    df_text = ''.join([format_text(i) for i in df_text])

                    df_output = "    " + \
                        format_chunk(df_chunk, 0, 4, chunk,
                                     args.decimal) + " | "
                    df_output += format_chunk(df_chunk,
                                              4, 8, chunk, args.decimal) + " | "
                    df_output += format_chunk(df_chunk,
                                              8, 12, chunk, args.decimal) + " | "
                    df_output += format_chunk(df_chunk, 12,
                                              16, chunk, args.decimal) + df_text

                if len(chunk) % b != 0:
                    if args.decimal:
                        output += "   " * (((b * 2) - 4 - len(chunk))) + text
                        if args.diff:
                            df_output += "   " * \
                                (((b * 2) - 4 - len(df_chunk))) + df_text
                    else:
                        output += "   " * (b + 4 - len(chunk)) + text
                        if args.diff:
                            df_output += "   " * \
                                (b + 4 - len(df_chunk)) + df_text
                else:
                    output += " " + text
                    if args.diff:
                        output += df_output

                print output
                offset += 16

            print
    else:
        print parser.usage
<|endoftext|>


<commit_before># Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import json
import os
import sys
from io import BytesIO

import jsonrpc  # pylint: disable=import-error
from twisted.internet import threads  # pylint: disable=import-error

from platformio import __main__, __version__, util
from platformio.compat import string_types

try:
    from thread import get_ident as thread_get_ident
except ImportError:
    from threading import get_ident as thread_get_ident

class MultiThreadingStdStream(object):

    def __init__(self, parent_stream):
        self._buffers = {thread_get_ident(): parent_stream}

    def __getattr__(self, name):
        thread_id = thread_get_ident()
        if thread_id not in self._buffers:
            raise AttributeError(name)
        return getattr(self._buffers[thread_id], name)

    def write(self, value):
        thread_id = thread_get_ident()
        if thread_id not in self._buffers:
            self._buffers[thread_id] = BytesIO()
        try:
            return self._buffers[thread_id].write(value)
        except TypeError:
            return self._buffers[thread_id].write(value.encode())

    def get_value_and_close(self):
        thread_id = thread_get_ident()
        result = ""
        try:
            result = self.getvalue()
            self.close()
            if thread_id in self._buffers:
                del self._buffers[thread_id]
        except AttributeError:
            pass
        return result

class PIOCoreRPC(object):

    @staticmethod
    def setup_multithreading_std_streams():
        if isinstance(sys.stdout, MultiThreadingStdStream):
            return
        PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout)
        PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr)
        sys.stdout = PIOCoreRPC.thread_stdout
        sys.stderr = PIOCoreRPC.thread_stderr

    @staticmethod
    def call(args, options=None):
        try:
            args = [
                str(arg) if not isinstance(arg, string_types) else arg
                for arg in args
            ]
        except UnicodeError:
            raise jsonrpc.exceptions.JSONRPCDispatchException(
                code=4002, message="PIO Core: non-ASCII chars in arguments")

        PIOCoreRPC.setup_multithreading_std_streams()
        cwd = (options or {}).get("cwd") or os.getcwd()

        def _call_inline():
            with util.cd(cwd):
                exit_code = __main__.main(["-c"] + args)
            return (PIOCoreRPC.thread_stdout.get_value_and_close(),
                    PIOCoreRPC.thread_stderr.get_value_and_close(), exit_code)

        d = threads.deferToThread(_call_inline)

        d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args)
        d.addErrback(PIOCoreRPC._call_errback)
        return d

    @staticmethod
    def _call_callback(result, json_output=False):
        out, err, code = result
        text = ("%s\n\n%s" % (out, err)).strip()
        if code != 0:
            raise Exception(text)
        return json.loads(out) if json_output else text

    @staticmethod
    def _call_errback(failure):
        raise jsonrpc.exceptions.JSONRPCDispatchException(
            code=4003,
            message="PIO Core Call Error",
            data=failure.getErrorMessage())

    @staticmethod
    def version():
        return __version__
<commit_msg>Minor changes<commit_after># Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import

import json
import os
import sys
from io import BytesIO

import jsonrpc  # pylint: disable=import-error
from twisted.internet import threads  # pylint: disable=import-error

from platformio import __main__, __version__, util
from platformio.compat import string_types

try:
    from thread import get_ident as thread_get_ident
except ImportError:
    from threading import get_ident as thread_get_ident

class MultiThreadingStdStream(object):

    def __init__(self, parent_stream):
        self._buffers = {thread_get_ident(): parent_stream}

    def __getattr__(self, name):
        thread_id = thread_get_ident()
        if thread_id not in self._buffers:
            raise AttributeError(name)
        return getattr(self._buffers[thread_id], name)

    def write(self, value):
        thread_id = thread_get_ident()
        if thread_id not in self._buffers:
            self._buffers[thread_id] = BytesIO()
        try:
            return self._buffers[thread_id].write(value)
        except TypeError:
            return self._buffers[thread_id].write(value.encode())

    def get_value_and_close(self):
        thread_id = thread_get_ident()
        result = ""
        try:
            result = self.getvalue()
            self.close()
            if thread_id in self._buffers:
                del self._buffers[thread_id]
        except AttributeError:
            pass
        return result

class PIOCoreRPC(object):

    @staticmethod
    def setup_multithreading_std_streams():
        if isinstance(sys.stdout, MultiThreadingStdStream):
            return
        PIOCoreRPC.thread_stdout = MultiThreadingStdStream(sys.stdout)
        PIOCoreRPC.thread_stderr = MultiThreadingStdStream(sys.stderr)
        sys.stdout = PIOCoreRPC.thread_stdout
        sys.stderr = PIOCoreRPC.thread_stderr

    @staticmethod
    def call(args, options=None):
        PIOCoreRPC.setup_multithreading_std_streams()
        try:
            args = [
                str(arg) if not isinstance(arg, string_types) else arg
                for arg in args
            ]
        except UnicodeError:
            raise jsonrpc.exceptions.JSONRPCDispatchException(
                code=4002, message="PIO Core: non-ASCII chars in arguments")

        def _call_inline():
            with util.cd((options or {}).get("cwd") or os.getcwd()):
                exit_code = __main__.main(["-c"] + args)
            return (PIOCoreRPC.thread_stdout.get_value_and_close(),
                    PIOCoreRPC.thread_stderr.get_value_and_close(), exit_code)

        d = threads.deferToThread(_call_inline)

        d.addCallback(PIOCoreRPC._call_callback, "--json-output" in args)
        d.addErrback(PIOCoreRPC._call_errback)
        return d

    @staticmethod
    def _call_callback(result, json_output=False):
        out, err, code = result
        text = ("%s\n\n%s" % (out, err)).strip()
        if code != 0:
            raise Exception(text)
        return json.loads(out) if json_output else text

    @staticmethod
    def _call_errback(failure):
        raise jsonrpc.exceptions.JSONRPCDispatchException(
            code=4003,
            message="PIO Core Call Error",
            data=failure.getErrorMessage())

    @staticmethod
    def version():
        return __version__
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division, absolute_import, unicode_literals, print_function

import argparse
import os.path
import shutil
import subprocess
import tempfile
import logging

import PyPDF2
import qrtools

log = logging.getLogger(__name__)

def parseCommandLine():
    parser = argparse.ArgumentParser(description='split pdf document based on separator pages with QR codes')
    group = parser.add_mutually_exclusive_group()
    group.add_argument('-v', '--verbose', action='store_true', help='be very verbose')
    group.add_argument('-q', '--quiet', action='store_true', help='no logging except errors')
    parser.add_argument('-d', '--delete-source', action='store_true', help='delete source document after processing')
    parser.add_argument('-g', '--ghostscript', action='store_true', help='executable to start ghostscript',
                        default='c:\\Program Files\\gs\\gs9.19\\bin\\gswin64c.exe')
    parser.add_argument('filename', help='path of the source document (PDF)')
    return parser.parse_args()

def extractPages(inputPDF, start, end, filename='pages from %s to %s.pdf'):
    log.debug('Extracting pages %s to %s to file %s', start, end, filename)
    output = PyPDF2.PdfFileWriter()
    for j in range(start, end):
        output.addPage(inputPDF.getPage(j))
    with open(filename % (start, end), "wb") as outputStream:
        output.write(outputStream)

def autoSplitPDF(filename, ghostScriptExecutable):
    qr = qrtools.QR()
    temporaryDirectory = tempfile.mkdtemp(prefix='autoSplitPDF - ')
    pngFiles = os.path.join(temporaryDirectory, 'pages-%03d.png')
    inputFile = os.path.abspath(filename)
    outputDirectory = os.path.dirname(filename)
    log.debug('Call ghostscript to split PDF into pages and export as PNG')
    subprocess.check_call([ghostScriptExecutable, '-dBATCH', '-dNOPAUSE', '-sDEVICE=pnggray', '-r300', '-dUseCropBox', '-sOutputFile=%s' % pngFiles, inputFile])
    log.debug('Opening %s as source PDF', inputFile)
    with open(inputFile, 'rb') as sourcePDF:
        reader = PyPDF2.PdfFileReader(sourcePDF)
        beginOfCurrentPageSet = 0
        files = []
        filenameOfCurrentPageSet = os.path.join(outputDirectory, 'Unknown (from %s to %s).pdf')
        for i in range(reader.numPages):
            log.debug('Trying to decode page %s (file %s)', i + 1, pngFiles % (i + 1))
            if qr.decode(pngFiles % (i + 1)) and qr.data.startswith('SEPARATOR SHEET'):
                log.info("Separator sheet with data: %s", qr.data)
                if i != beginOfCurrentPageSet:
                    files += [extractPages(reader, beginOfCurrentPageSet, i, filenameOfCurrentPageSet)]
                filenameOfCurrentPageSet = os.path.join(outputDirectory, qr.data.split('-', 1)[-1].strip() + ' (from %s to %s).pdf')
                beginOfCurrentPageSet = i + 1
        if beginOfCurrentPageSet < reader.numPages:
            extractPages(reader, beginOfCurrentPageSet, reader.numPages, filenameOfCurrentPageSet)
        shutil.rmtree(temporaryDirectory)

if __name__ == '__main__':
    args = parseCommandLine()
    if args.verbose:
        log.setLevel(logging.DEBUG)
    if args.quiet:
        log.setLevel(logging.ERROR)
    autoSplitPDF(args.filename, args.ghostscript)
    if args.delete_source:
        os.remove(args.filename)
<commit_msg>logging config<commit_after># -*- coding: utf-8 -*-
from __future__ import division, absolute_import, unicode_literals, print_function

import argparse
import os.path
import shutil
import subprocess
import tempfile
import logging

import PyPDF2
import qrtools

logging.basicConfig(level=logging.INFO, format='%(asctime)s,%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s')
log = logging.getLogger(__name__)

def parseCommandLine():
    parser = argparse.ArgumentParser(description='split pdf document based on separator pages with QR codes')
    group = parser.add_mutually_exclusive_group()
    group.add_argument('-v', '--verbose', action='store_true', help='be very verbose')
    group.add_argument('-q', '--quiet', action='store_true', help='no logging except errors')
    parser.add_argument('-d', '--delete-source', action='store_true', help='delete source document after processing')
    parser.add_argument('-g', '--ghostscript', action='store_true', help='executable to start ghostscript',
                        default='c:\\Program Files\\gs\\gs9.19\\bin\\gswin64c.exe')
    parser.add_argument('filename', help='path of the source document (PDF)')
    return parser.parse_args()

def extractPages(inputPDF, start, end, filename='pages from %s to %s.pdf'):
    log.debug('Extracting pages %s to %s to file %s', start, end, filename)
    output = PyPDF2.PdfFileWriter()
    for j in range(start, end):
        output.addPage(inputPDF.getPage(j))
    with open(filename % (start, end), "wb") as outputStream:
        output.write(outputStream)

def autoSplitPDF(filename, ghostScriptExecutable):
    qr = qrtools.QR()
    temporaryDirectory = tempfile.mkdtemp(prefix='autoSplitPDF - ')
    pngFiles = os.path.join(temporaryDirectory, 'pages-%03d.png')
    inputFile = os.path.abspath(filename)
    outputDirectory = os.path.dirname(filename)
    log.debug('Call ghostscript to split PDF (%s) into pages and export as PNG (%s)', inputFile, pngFiles)
    subprocess.check_call([ghostScriptExecutable, '-dBATCH', '-dNOPAUSE', '-sDEVICE=pnggray', '-r300', '-dUseCropBox', '-sOutputFile=%s' % pngFiles, inputFile])
    log.debug('Opening %s as source PDF', inputFile)
    with open(inputFile, 'rb') as sourcePDF:
        reader = PyPDF2.PdfFileReader(sourcePDF)
        beginOfCurrentPageSet = 0
        files = []
        filenameOfCurrentPageSet = os.path.join(outputDirectory, 'Unknown (from %s to %s).pdf')
        for i in range(reader.numPages):
            log.debug('Trying to decode page %s (file %s)', i + 1, pngFiles % (i + 1))
            if qr.decode(pngFiles % (i + 1)) and qr.data.startswith('SEPARATOR SHEET'):
                log.info("Separator sheet with data: %s", qr.data)
                if i != beginOfCurrentPageSet:
                    files += [extractPages(reader, beginOfCurrentPageSet, i, filenameOfCurrentPageSet)]
                filenameOfCurrentPageSet = os.path.join(outputDirectory, qr.data.split('-', 1)[-1].strip() + ' (from %s to %s).pdf')
                beginOfCurrentPageSet = i + 1
        if beginOfCurrentPageSet < reader.numPages:
            extractPages(reader, beginOfCurrentPageSet, reader.numPages, filenameOfCurrentPageSet)
        shutil.rmtree(temporaryDirectory)

if __name__ == '__main__':
    args = parseCommandLine()
    if args.verbose:
        log.setLevel(logging.DEBUG)
    if args.quiet:
        log.setLevel(logging.ERROR)
    autoSplitPDF(args.filename, args.ghostscript)
    if args.delete_source:
        os.remove(args.filename)
<|endoftext|>


<commit_before>from typing import List, NamedTuple, Optional

from plenum.common.config_util import getConfig
from plenum.common.messages.node_messages import PrePrepare, Checkpoint
from sortedcontainers import SortedListWithKey

from plenum.common.startable import Mode
from plenum.common.util import SortedDict
from plenum.server.consensus.view_change_storages import ViewChangeVotesForView
from plenum.server.models import Prepares, Commits
from plenum.common.types import f
from plenum.server.propagator import Requests
from plenum.server.quorums import Quorums

# `view_no` is a view no is the current view_no, but `pp_view_no` is a view no when the given PrePrepare has been
# initially created and applied

# it's critical to keep the original view no to correctly create audit ledger transaction
# (since PrePrepare's view no is present there)

# An example when `view_no` != `pp_view_no`, is when view change didn't finish at first round
# (next primary is unavailable for example)
BatchID = NamedTuple('BatchID', [('view_no', int), ('pp_view_no', int), ('pp_seq_no', int), ('pp_digest', str)])

def get_original_viewno(pp):
    return pp.originalViewNo if f.ORIGINAL_VIEW_NO.nm in pp else pp.viewNo

def preprepare_to_batch_id(pre_prepare: PrePrepare) -> BatchID:
    pp_view_no = get_original_viewno(pre_prepare)
    return BatchID(pre_prepare.viewNo, pp_view_no, pre_prepare.ppSeqNo, pre_prepare.digest)

class ConsensusSharedData:
    """
    This is a 3PC-state shared between Ordering, Checkpoint and ViewChange services.
    TODO: Consider depending on audit ledger
    TODO: Consider adding persistent local storage for 3PC certificates
    TODO: Restore primary name from audit ledger instead of passing through constructor
    """

    def __init__(self, name: str, validators: List[str], inst_id: int, is_master: bool = True):
        self._name = name
        self.inst_id = inst_id
        self.view_no = 0
        self.waiting_for_new_view = False
        self.primaries = []
        self.is_master = is_master

        self.legacy_vc_in_progress = False
        self.requests = Requests()
        self.last_ordered_3pc = (0, 0)
        # Indicates name of the primary replica of this protocol instance.
        # None in case the replica does not know who the primary of the
        # instance is
        # TODO: Replace this by read-only property which uses primaries and inst_id
        self.primary_name = None
        # seqNoEnd of the last stabilized checkpoint
        self.stable_checkpoint = 0
        # Checkpoint messages which the current node sent.
        # TODO: Replace sorted list with dict
        self.checkpoints = SortedListWithKey(key=lambda checkpoint: checkpoint.seqNoEnd)
        self.checkpoints.append(self.initial_checkpoint)
        # List of BatchIDs of PrePrepare messages for which quorum of Prepare messages is not reached yet
        self.preprepared = []  # type:  List[BatchID]
        # List of BatchIDs of PrePrepare messages for which quorum of Prepare messages is reached
        self.prepared = []  # type:  List[BatchID]
        self._validators = None
        self.quorums = None
        self.view_change_votes = ViewChangeVotesForView(self.quorums)
        # a list of validator node names ordered by rank (historical order of adding)
        self.set_validators(validators)
        self.low_watermark = 0
        self.log_size = getConfig().LOG_SIZE
        self.high_watermark = self.low_watermark + self.log_size
        self.pp_seq_no = 0
        self.node_mode = Mode.starting
        # ToDo: it should be set in view_change_service before view_change starting
        # 3 phase key for the last prepared certificate before view change
        # started, applicable only to master instance
        self.legacy_last_prepared_before_view_change = None
        self.prev_view_prepare_cert = None

        # Dictionary of sent PRE-PREPARE that are stored by primary replica
        # which it has broadcasted to all other non primary replicas
        # Key of dictionary is a 2 element tuple with elements viewNo,
        # pre-prepare seqNo and value is the received PRE-PREPARE
        self.sent_preprepares = SortedDict(lambda k: (k[0], k[1]))
        # type: Dict[Tuple[int, int], PrePrepare]

        # Dictionary of all Prepare requests. Key of dictionary is a 2
        # element tuple with elements viewNo, seqNo and value is a 2 element
        # tuple containing request digest and set of sender node names(sender
        # replica names in case of multiple protocol instances)
        # (viewNo, seqNo) -> ((identifier, reqId), {senders})
        self.prepares = Prepares()
        # type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]

        self.commits = Commits()
        # type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]

        # Tracks for which keys PRE-PREPAREs have been requested.
        # Cleared in `gc`
        self.requested_pre_prepares = {}

    @property
    def name(self) -> str:
        return self._name

    def set_validators(self, validators: List[str]):
        self._validators = validators
        self.quorums = Quorums(len(validators))
        self.view_change_votes.update_quorums(self.quorums)

    @property
    def validators(self) -> List[str]:
        """
        List of validator nodes aliases
        """
        return self._validators

    @property
    def is_primary(self) -> Optional[bool]:
        """
        TODO: It would be much more clear and easy to use if this returned just bool.
        Returns is replica primary for this instance.
        If primary name is not defined yet, returns None
        """
        return None if self.primary_name is None else self.primary_name == self.name

    @property
    def is_participating(self):
        return self.node_mode == Mode.participating

    @property
    def is_synced(self):
        return Mode.is_done_syncing(self.node_mode)

    @property
    def total_nodes(self):
        return len(self.validators)

    @property
    def initial_checkpoint(self):
        return Checkpoint(instId=self.inst_id, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=None)

    @property
    def last_checkpoint(self) -> Checkpoint:
        return self.checkpoints[-1]
<commit_msg>INDY-2178: fix a problem with ViewChangeVotesForView creation<commit_after>from typing import List, NamedTuple, Optional

from plenum.common.config_util import getConfig
from plenum.common.messages.node_messages import PrePrepare, Checkpoint
from sortedcontainers import SortedListWithKey

from plenum.common.startable import Mode
from plenum.common.util import SortedDict
from plenum.server.consensus.view_change_storages import ViewChangeVotesForView
from plenum.server.models import Prepares, Commits
from plenum.common.types import f
from plenum.server.propagator import Requests
from plenum.server.quorums import Quorums

# `view_no` is a view no is the current view_no, but `pp_view_no` is a view no when the given PrePrepare has been
# initially created and applied

# it's critical to keep the original view no to correctly create audit ledger transaction
# (since PrePrepare's view no is present there)

# An example when `view_no` != `pp_view_no`, is when view change didn't finish at first round
# (next primary is unavailable for example)
BatchID = NamedTuple('BatchID', [('view_no', int), ('pp_view_no', int), ('pp_seq_no', int), ('pp_digest', str)])

def get_original_viewno(pp):
    return pp.originalViewNo if f.ORIGINAL_VIEW_NO.nm in pp else pp.viewNo

def preprepare_to_batch_id(pre_prepare: PrePrepare) -> BatchID:
    pp_view_no = get_original_viewno(pre_prepare)
    return BatchID(pre_prepare.viewNo, pp_view_no, pre_prepare.ppSeqNo, pre_prepare.digest)

class ConsensusSharedData:
    """
    This is a 3PC-state shared between Ordering, Checkpoint and ViewChange services.
    TODO: Consider depending on audit ledger
    TODO: Consider adding persistent local storage for 3PC certificates
    TODO: Restore primary name from audit ledger instead of passing through constructor
    """

    def __init__(self, name: str, validators: List[str], inst_id: int, is_master: bool = True):
        self._name = name
        self.inst_id = inst_id
        self.view_no = 0
        self.waiting_for_new_view = False
        self.primaries = []
        self.is_master = is_master

        self.legacy_vc_in_progress = False
        self.requests = Requests()
        self.last_ordered_3pc = (0, 0)
        # Indicates name of the primary replica of this protocol instance.
        # None in case the replica does not know who the primary of the
        # instance is
        # TODO: Replace this by read-only property which uses primaries and inst_id
        self.primary_name = None
        # seqNoEnd of the last stabilized checkpoint
        self.stable_checkpoint = 0
        # Checkpoint messages which the current node sent.
        # TODO: Replace sorted list with dict
        self.checkpoints = SortedListWithKey(key=lambda checkpoint: checkpoint.seqNoEnd)
        self.checkpoints.append(self.initial_checkpoint)
        # List of BatchIDs of PrePrepare messages for which quorum of Prepare messages is not reached yet
        self.preprepared = []  # type:  List[BatchID]
        # List of BatchIDs of PrePrepare messages for which quorum of Prepare messages is reached
        self.prepared = []  # type:  List[BatchID]
        self._validators = None
        self.quorums = None
        self.view_change_votes = ViewChangeVotesForView(Quorums(len(validators)))
        # a list of validator node names ordered by rank (historical order of adding)
        self.set_validators(validators)
        self.low_watermark = 0
        self.log_size = getConfig().LOG_SIZE
        self.high_watermark = self.low_watermark + self.log_size
        self.pp_seq_no = 0
        self.node_mode = Mode.starting
        # ToDo: it should be set in view_change_service before view_change starting
        # 3 phase key for the last prepared certificate before view change
        # started, applicable only to master instance
        self.legacy_last_prepared_before_view_change = None
        self.prev_view_prepare_cert = None

        # Dictionary of sent PRE-PREPARE that are stored by primary replica
        # which it has broadcasted to all other non primary replicas
        # Key of dictionary is a 2 element tuple with elements viewNo,
        # pre-prepare seqNo and value is the received PRE-PREPARE
        self.sent_preprepares = SortedDict(lambda k: (k[0], k[1]))
        # type: Dict[Tuple[int, int], PrePrepare]

        # Dictionary of all Prepare requests. Key of dictionary is a 2
        # element tuple with elements viewNo, seqNo and value is a 2 element
        # tuple containing request digest and set of sender node names(sender
        # replica names in case of multiple protocol instances)
        # (viewNo, seqNo) -> ((identifier, reqId), {senders})
        self.prepares = Prepares()
        # type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]

        self.commits = Commits()
        # type: Dict[Tuple[int, int], Tuple[Tuple[str, int], Set[str]]]

        # Tracks for which keys PRE-PREPAREs have been requested.
        # Cleared in `gc`
        self.requested_pre_prepares = {}

    @property
    def name(self) -> str:
        return self._name

    def set_validators(self, validators: List[str]):
        self._validators = validators
        self.quorums = Quorums(len(validators))
        self.view_change_votes.update_quorums(self.quorums)

    @property
    def validators(self) -> List[str]:
        """
        List of validator nodes aliases
        """
        return self._validators

    @property
    def is_primary(self) -> Optional[bool]:
        """
        TODO: It would be much more clear and easy to use if this returned just bool.
        Returns is replica primary for this instance.
        If primary name is not defined yet, returns None
        """
        return None if self.primary_name is None else self.primary_name == self.name

    @property
    def is_participating(self):
        return self.node_mode == Mode.participating

    @property
    def is_synced(self):
        return Mode.is_done_syncing(self.node_mode)

    @property
    def total_nodes(self):
        return len(self.validators)

    @property
    def initial_checkpoint(self):
        return Checkpoint(instId=self.inst_id, viewNo=0, seqNoStart=0, seqNoEnd=0, digest=None)

    @property
    def last_checkpoint(self) -> Checkpoint:
        return self.checkpoints[-1]
<|endoftext|>


<commit_before>from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils.translation import ugettext as _
from django.conf import settings

from django.contrib.auth.decorators import login_required

from avatar.forms import PrimaryAvatarForm, DeleteAvatarForm, UploadAvatarForm
from avatar.models import Avatar
from avatar.settings import AVATAR_MAX_AVATARS_PER_USER, AVATAR_DEFAULT_SIZE
from avatar.signals import avatar_updated
from avatar.util import get_primary_avatar, get_default_avatar_url

def _get_next(request):
    """
    The part that's the least straightforward about views in this module is how they
    determine their redirects after they have finished computation.

    In short, they will try and determine the next place to go in the following order:

    1. If there is a variable named ``next`` in the *POST* parameters, the view will
    redirect to that variable's value.
    2. If there is a variable named ``next`` in the *GET* parameters, the view will
    redirect to that variable's value.
    3. If Django can determine the previous page from the HTTP headers, the view will
    redirect to that previous page.
    """
    next = request.POST.get('next', request.GET.get('next',
        request.META.get('HTTP_REFERER', None)))
    if not next:
        next = request.path
    return next

def _get_avatars(user):
    # Default set. Needs to be sliced, but that's it. Keep the natural order.
    avatars = user.avatar_set.all()

    # Current avatar
    primary_avatar = avatars.order_by('-primary')[:1]
    if primary_avatar:
        avatar = primary_avatar[0]
    else:
        avatar = None

    if AVATAR_MAX_AVATARS_PER_USER == 1:
        avatars = primary_avatar
    else:
        # Slice the default set now that we used the queryset for the primary avatar
        avatars = avatars[:AVATAR_MAX_AVATARS_PER_USER]
    return (avatar, avatars)

@login_required
def add(request, extra_context=None, next_override=None,
        upload_form=UploadAvatarForm, *args, **kwargs):
    if extra_context is None:
        extra_context = {}
    avatar, avatars = _get_avatars(request.user)
    upload_avatar_form = upload_form(request.POST or None,
        request.FILES or None, user=request.user)
    if request.method == "POST" and 'avatar' in request.FILES:
        if upload_avatar_form.is_valid():
            avatar = Avatar(
                user = request.user,
                primary = True,
            )
            image_file = request.FILES['avatar']
            avatar.avatar.save(image_file.name, image_file)
            avatar.save()
            request.user.message_set.create(
                message=_("Successfully uploaded a new avatar."))
            avatar_updated.send(sender=Avatar, user=request.user, avatar=avatar)
            return HttpResponseRedirect(next_override or _get_next(request))
    return render_to_response(
            'avatar/add.html',
            extra_context,
            context_instance = RequestContext(
                request,
                { 'avatar': avatar,
                  'avatars': avatars,
                  'upload_avatar_form': upload_avatar_form,
                  'next': next_override or _get_next(request), }
            )
        )

@login_required
def change(request, extra_context=None, next_override=None,
        upload_form=UploadAvatarForm, primary_form=PrimaryAvatarForm,
        *args, **kwargs):
    if extra_context is None:
        extra_context = {}
    avatar, avatars = _get_avatars(request.user)
    if avatar:
        kwargs = {'initial': {'choice': avatar.id}}
    else:
        kwargs = {}
    upload_avatar_form = upload_form(user=request.user, **kwargs)
    primary_avatar_form = primary_form(request.POST or None,
        user=request.user, avatars=avatars, **kwargs)
    if request.method == "POST":
        updated = False
        if 'choice' in request.POST and primary_avatar_form.is_valid():
            avatar = Avatar.objects.get(id=
                primary_avatar_form.cleaned_data['choice'])
            avatar.primary = True
            avatar.save()
            updated = True
            request.user.message_set.create(
                message=_("Successfully updated your avatar."))
        if updated:
            avatar_updated.send(sender=Avatar, user=request.user, avatar=avatar)
        return HttpResponseRedirect(next_override or _get_next(request))
    return render_to_response(
        'avatar/change.html',
        extra_context,
        context_instance = RequestContext(
            request,
            { 'avatar': avatar,
              'avatars': avatars,
              'upload_avatar_form': upload_avatar_form,
              'primary_avatar_form': primary_avatar_form,
              'next': next_override or _get_next(request), }
        )
    )

@login_required
def delete(request, extra_context=None, next_override=None, *args, **kwargs):
    if extra_context is None:
        extra_context = {}
    avatar, avatars = _get_avatars(request.user)
    delete_avatar_form = DeleteAvatarForm(request.POST or None,
        user=request.user, avatars=avatars)
    if request.method == 'POST':
        if delete_avatar_form.is_valid():
            ids = delete_avatar_form.cleaned_data['choices']
            if unicode(avatar.id) in ids and avatars.count() > len(ids):
                # Find the next best avatar, and set it as the new primary
                for a in avatars:
                    if unicode(a.id) not in ids:
                        a.primary = True
                        a.save()
                        avatar_updated.send(sender=Avatar, user=request.user, avatar=avatar)
                        break
            Avatar.objects.filter(id__in=ids).delete()
            request.user.message_set.create(
                message=_("Successfully deleted the requested avatars."))
            return HttpResponseRedirect(next_override or _get_next(request))
    return render_to_response(
        'avatar/confirm_delete.html',
        extra_context,
        context_instance = RequestContext(
            request,
            { 'avatar': avatar,
              'avatars': avatars,
              'delete_avatar_form': delete_avatar_form,
              'next': next_override or _get_next(request), }
        )
    )

def render_primary(request, extra_context={}, user=None, size=AVATAR_DEFAULT_SIZE, *args, **kwargs):
    size = int(size)
    avatar = get_primary_avatar(user, size=size)
    if avatar:
        # FIXME: later, add an option to render the resized avatar dynamically
        # instead of redirecting to an already created static file. This could
        # be useful in certain situations, particulary if there is a CDN and
        # we want to minimize the storage usage on our static server, letting
        # the CDN store those files instead
        return HttpResponseRedirect(avatar.avatar_url(size))
    else:
        url = get_default_avatar_url()
        return HttpResponseRedirect(url)

<commit_msg>added template_name argument to change view<commit_after>from django.http import HttpResponseRedirect
from django.shortcuts import render_to_response
from django.template import RequestContext
from django.utils.translation import ugettext as _
from django.conf import settings

from django.contrib.auth.decorators import login_required

from avatar.forms import PrimaryAvatarForm, DeleteAvatarForm, UploadAvatarForm
from avatar.models import Avatar
from avatar.settings import AVATAR_MAX_AVATARS_PER_USER, AVATAR_DEFAULT_SIZE
from avatar.signals import avatar_updated
from avatar.util import get_primary_avatar, get_default_avatar_url

def _get_next(request):
    """
    The part that's the least straightforward about views in this module is how they
    determine their redirects after they have finished computation.

    In short, they will try and determine the next place to go in the following order:

    1. If there is a variable named ``next`` in the *POST* parameters, the view will
    redirect to that variable's value.
    2. If there is a variable named ``next`` in the *GET* parameters, the view will
    redirect to that variable's value.
    3. If Django can determine the previous page from the HTTP headers, the view will
    redirect to that previous page.
    """
    next = request.POST.get('next', request.GET.get('next',
        request.META.get('HTTP_REFERER', None)))
    if not next:
        next = request.path
    return next

def _get_avatars(user):
    # Default set. Needs to be sliced, but that's it. Keep the natural order.
    avatars = user.avatar_set.all()

    # Current avatar
    primary_avatar = avatars.order_by('-primary')[:1]
    if primary_avatar:
        avatar = primary_avatar[0]
    else:
        avatar = None

    if AVATAR_MAX_AVATARS_PER_USER == 1:
        avatars = primary_avatar
    else:
        # Slice the default set now that we used the queryset for the primary avatar
        avatars = avatars[:AVATAR_MAX_AVATARS_PER_USER]
    return (avatar, avatars)

@login_required
def add(request, extra_context=None, next_override=None,
        upload_form=UploadAvatarForm, *args, **kwargs):
    if extra_context is None:
        extra_context = {}
    avatar, avatars = _get_avatars(request.user)
    upload_avatar_form = upload_form(request.POST or None,
        request.FILES or None, user=request.user)
    if request.method == "POST" and 'avatar' in request.FILES:
        if upload_avatar_form.is_valid():
            avatar = Avatar(
                user = request.user,
                primary = True,
            )
            image_file = request.FILES['avatar']
            avatar.avatar.save(image_file.name, image_file)
            avatar.save()
            request.user.message_set.create(
                message=_("Successfully uploaded a new avatar."))
            avatar_updated.send(sender=Avatar, user=request.user, avatar=avatar)
            return HttpResponseRedirect(next_override or _get_next(request))
    return render_to_response(
            'avatar/add.html',
            extra_context,
            context_instance = RequestContext(
                request,
                { 'avatar': avatar,
                  'avatars': avatars,
                  'upload_avatar_form': upload_avatar_form,
                  'next': next_override or _get_next(request), }
            )
        )

@login_required
def change(request, extra_context=None, next_override=None,
        upload_form=UploadAvatarForm, primary_form=PrimaryAvatarForm,
        template_name='avatar/change.html',
        *args, **kwargs):
    if extra_context is None:
        extra_context = {}
    avatar, avatars = _get_avatars(request.user)
    if avatar:
        kwargs = {'initial': {'choice': avatar.id}}
    else:
        kwargs = {}
    upload_avatar_form = upload_form(user=request.user, **kwargs)
    primary_avatar_form = primary_form(request.POST or None,
        user=request.user, avatars=avatars, **kwargs)
    if request.method == "POST":
        updated = False
        if 'choice' in request.POST and primary_avatar_form.is_valid():
            avatar = Avatar.objects.get(id=
                primary_avatar_form.cleaned_data['choice'])
            avatar.primary = True
            avatar.save()
            updated = True
            request.user.message_set.create(
                message=_("Successfully updated your avatar."))
        if updated:
            avatar_updated.send(sender=Avatar, user=request.user, avatar=avatar)
        return HttpResponseRedirect(next_override or _get_next(request))
    return render_to_response(
        template_name,
        extra_context,
        context_instance = RequestContext(
            request,
            { 'avatar': avatar,
              'avatars': avatars,
              'upload_avatar_form': upload_avatar_form,
              'primary_avatar_form': primary_avatar_form,
              'next': next_override or _get_next(request), }
        )
    )

@login_required
def delete(request, extra_context=None, next_override=None, *args, **kwargs):
    if extra_context is None:
        extra_context = {}
    avatar, avatars = _get_avatars(request.user)
    delete_avatar_form = DeleteAvatarForm(request.POST or None,
        user=request.user, avatars=avatars)
    if request.method == 'POST':
        if delete_avatar_form.is_valid():
            ids = delete_avatar_form.cleaned_data['choices']
            if unicode(avatar.id) in ids and avatars.count() > len(ids):
                # Find the next best avatar, and set it as the new primary
                for a in avatars:
                    if unicode(a.id) not in ids:
                        a.primary = True
                        a.save()
                        avatar_updated.send(sender=Avatar, user=request.user, avatar=avatar)
                        break
            Avatar.objects.filter(id__in=ids).delete()
            request.user.message_set.create(
                message=_("Successfully deleted the requested avatars."))
            return HttpResponseRedirect(next_override or _get_next(request))
    return render_to_response(
        'avatar/confirm_delete.html',
        extra_context,
        context_instance = RequestContext(
            request,
            { 'avatar': avatar,
              'avatars': avatars,
              'delete_avatar_form': delete_avatar_form,
              'next': next_override or _get_next(request), }
        )
    )

def render_primary(request, extra_context={}, user=None, size=AVATAR_DEFAULT_SIZE, *args, **kwargs):
    size = int(size)
    avatar = get_primary_avatar(user, size=size)
    if avatar:
        # FIXME: later, add an option to render the resized avatar dynamically
        # instead of redirecting to an already created static file. This could
        # be useful in certain situations, particulary if there is a CDN and
        # we want to minimize the storage usage on our static server, letting
        # the CDN store those files instead
        return HttpResponseRedirect(avatar.avatar_url(size))
    else:
        url = get_default_avatar_url()
        return HttpResponseRedirect(url)

<|endoftext|>


<commit_before>#
<commit_msg>Exposing HITC and Model objects through hitcpy namespace.<commit_after>from hitcpy import HITC, Model
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Functions for building sphinx docs.

For more information on the invocation of sphinx see:
http://sphinx-doc.org/invocation.html
"""
import subprocess
import os
import sys
import fnmatch
import shutil
import imp
from glob import glob

from distutils.dir_util import copy_tree
from distutils.util import get_platform
from shutil import copy
from os import listdir

platform = '.%s-%s'%(get_platform(),sys.version[:3])

CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))

run = imp.load_source('run', os.path.join(CURRENT_SCRIPT_DIR, '..', '..', 'run.py'))
run.prepare()

SASVIEW_SRC = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "src")
SASVIEW_BUILD = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "build", "lib"+platform))
SASVIEW_DOCS = os.path.join(SASVIEW_BUILD, "doc")
SASVIEW_TEST = os.path.join(SASVIEW_SRC, "..", "sasview", "test", "media")

# Need to slurp in the new sasmodels model definitions to replace the old model_functions.rst
# We are currently here:
#/sasview-local-trunk/docs/sphinx-docs/build_sphinx.py
SASMODELS_SOURCE_PROLOG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc")
SASMODELS_SOURCE_REF_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "ref", "models")
SASMODELS_SOURCE_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "model")
SASMODELS_SOURCE_IMG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "model", "img")
SASMODELS_DEST_PROLOG = os.path.join(CURRENT_SCRIPT_DIR, "source")
SASMODELS_DEST_REF_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "source", "user")
SASMODELS_DEST_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "source", "user", "models")
SASMODELS_DEST_IMG = os.path.join(CURRENT_SCRIPT_DIR,  "source", "user", "model-imgs", "new-models")

#if os.path.exists(SASMODELS_SOURCE_PROLOG):
#    print "Found models prolog folder at ", SASMODELS_SOURCE_PROLOG
#if os.path.exists(SASMODELS_SOURCE_REF_MODELS):
#    print "Found models ref folder at ", SASMODELS_SOURCE_REF_MODELS
#if os.path.exists(SASMODELS_SOURCE_MODELS):
#    print "Found models folder at ", SASMODELS_SOURCE_MODELS
#if os.path.exists(SASMODELS_SOURCE_IMG):
#    print "Found img folder at ", SASMODELS_SOURCE_IMG
#if os.path.exists(SASMODELS_DEST_REF_MODELS):
#    print "Found models ref folder at ", SASMODELS_DEST_REF_MODELS
#if os.path.exists(SASMODELS_DEST_MODELS):
#    print "Found models folder at ", SASMODELS_DEST_MODELS
#if os.path.exists(SASMODELS_DEST_IMG):
#    print "Found img folder at ", SASMODELS_DEST_IMG
#sys.exit()

SPHINX_BUILD = os.path.join(CURRENT_SCRIPT_DIR, "build")
SPHINX_SOURCE = os.path.join(CURRENT_SCRIPT_DIR, "source")
SPHINX_SOURCE_API = os.path.join(SPHINX_SOURCE, "dev", "api")
SPHINX_SOURCE_GUIFRAME = os.path.join(SPHINX_SOURCE, "user", "sasgui", "guiframe")
SPHINX_SOURCE_MODELS = os.path.join(SPHINX_SOURCE, "user", "models")
SPHINX_SOURCE_PERSPECTIVES = os.path.join(SPHINX_SOURCE, "user", "sasgui", "perspectives")
SPHINX_SOURCE_TEST = os.path.join(SPHINX_SOURCE, "test")

BUMPS_DOCS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..",
                          "bumps", "doc", "guide")
BUMPS_TARGET = os.path.join(SPHINX_SOURCE_PERSPECTIVES, "fitting")

def inplace_change(filename, old_string, new_string):
# Thanks to http://stackoverflow.com/questions/4128144/replace-string-within-file-contents
        s=open(filename).read()
        if old_string in s:
                print 'Changing "{old_string}" to "{new_string}"'.format(**locals())
                s=s.replace(old_string, new_string)
                f=open(filename, 'w')
                f.write(s)
                f.flush()
                f.close()
        else:
                print 'No occurences of "{old_string}" found.'.format(**locals())

def _remove_dir(dir_path):
    """Removes the given directory."""
    if os.path.isdir(dir_path):
        print "Removing \"%s\"... " % dir_path
        shutil.rmtree(dir_path)

def clean():
    """
    Clean the sphinx build directory.
    """
    print "=== Cleaning Sphinx Build ==="
    _remove_dir(SASVIEW_DOCS)
    _remove_dir(SPHINX_BUILD)
    _remove_dir(SPHINX_SOURCE_GUIFRAME)
    _remove_dir(SPHINX_SOURCE_MODELS)
    _remove_dir(SPHINX_SOURCE_PERSPECTIVES)
    _remove_dir(SPHINX_SOURCE_TEST)

def retrieve_user_docs():
    """
    Copies across the contents of any media/ directories in src/, and puts them
    in an appropriately named directory of docs/sphinx-docs/source/. For
    example:

        sas/../[MODULE]/media/dir/A.rst
        sas/../[MODULE]/media/B.rst

    gets copied to a new location:

        docs/sphinx-docs/source/user/[MODULE]/dir/A.rst
        docs/sphinx-docs/source/user/[MODULE]/B.rst

    so that Sphinx may pick it up when generating the documentation.
    """
    print "=== Retrieve User Docs ==="

    # Copy documentation files from their "source" to their "destination".
    for root, dirnames, _ in os.walk(SASVIEW_SRC):
        for dirname in fnmatch.filter(dirnames, 'media'):

            docs = os.path.abspath(os.path.join(root, dirname))
            print "Found docs folder at \"%s\"." % docs

            dest_dir_part = os.path.dirname(os.path.relpath(docs, SASVIEW_SRC))
            if os.sep in dest_dir_part:
                dest_dir_part = dest_dir_part[dest_dir_part.index(os.sep) + 1:]
            dest_dir = os.path.join(SPHINX_SOURCE, "user", dest_dir_part)

            copy_tree(docs, dest_dir)

    # Now pickup testdata_help.rst
#    print os.path.abspath(SASVIEW_TEST)
#    print os.path.abspath(SPHINX_SOURCE_TEST)
    print "=== Including Test Data Docs ==="
    if os.path.exists(SASVIEW_TEST):
       print "Found docs folder at ", SASVIEW_TEST
       shutil.copytree(SASVIEW_TEST, SPHINX_SOURCE_TEST)

    print "=== And the Sasmodels Docs ==="
    # Make sure we have the relevant images for the new sasmodels documentation
    if os.path.exists(SASMODELS_SOURCE_IMG):
        print "Found img  folder SASMODELS_SOURCE_IMG    at ", SASMODELS_SOURCE_IMG
        if os.path.exists(SASMODELS_DEST_IMG):
            print "Found img  folder SASMODELS_DEST_IMG      at ", SASMODELS_DEST_IMG
            print "Copying sasmodels model image files..."
            for files in os.listdir(SASMODELS_SOURCE_IMG):
                fromhere=os.path.join(SASMODELS_SOURCE_IMG,files)
                tohere=os.path.join(SASMODELS_DEST_IMG,files)
                shutil.copy(fromhere,tohere)

    # And the rst prolog with the unit substitutions
    if os.path.exists(SASMODELS_SOURCE_PROLOG):
        print "Found prolog folder SASMODELS_SOURCE_PROLOG at ", SASMODELS_SOURCE_PROLOG
        if os.path.exists(SASMODELS_DEST_PROLOG):
            print "Found docs folder SASMODELS_DEST_PROLOG   at ", SASMODELS_DEST_PROLOG
            print "Copying sasmodels rst_prolog file..."
            for files in os.listdir(SASMODELS_SOURCE_PROLOG):
                if files.startswith("rst"):
                    fromhere=os.path.join(SASMODELS_SOURCE_PROLOG,files)
                    tohere=os.path.join(SASMODELS_DEST_PROLOG,files)
                    shutil.copy(fromhere,tohere)

    if os.path.exists(SASMODELS_SOURCE_REF_MODELS):
        print "Found docs folder SASMODELS_SOURCE_REF_MODELS at ", SASMODELS_SOURCE_REF_MODELS
        if os.path.exists(SASMODELS_DEST_REF_MODELS):
            print "Found docs folder SASMODELS_DEST_REF_MODELS   at ", SASMODELS_DEST_REF_MODELS
            print "Copying sasmodels model toctree files..."
            for files in os.listdir(SASMODELS_SOURCE_REF_MODELS):
                if files.endswith(".rst"):
                    fromhere=os.path.join(SASMODELS_SOURCE_REF_MODELS,files)
                    tohere=os.path.join(SASMODELS_DEST_REF_MODELS,files)
                    shutil.copy(fromhere,tohere)
    # But need to change the path to the model docs in the tocs
    for files in os.listdir(SASMODELS_DEST_REF_MODELS):
#        print files
        if files.startswith("shape"):
            print "Changing toc paths in", files
            inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS,files), "../../model/", "models/")
        if files.startswith("sphere"):
            print "Changing toc paths in", files
            inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS,files), "../../model/", "models/")
        if files.startswith("custom"):
            print "Changing toc paths in", files
            inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS,files), "../../model/", "models/")
        if files.startswith("structure"):
            print "Changing toc paths in", files
            inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS,files), "../../model/", "models/")

    if os.path.exists(SASMODELS_SOURCE_MODELS):
        print "Found docs folder SASMODELS_SOURCE_MODELS at ", SASMODELS_SOURCE_MODELS
        if os.path.exists(SASMODELS_DEST_MODELS):
            print "Found docs folder SASMODELS_DEST_MODELS   at ", SASMODELS_DEST_MODELS
            print "Copying sasmodels model files..."
            for files in os.listdir(SASMODELS_SOURCE_MODELS):
                if files.endswith(".rst"):
                    fromhere=os.path.join(SASMODELS_SOURCE_MODELS,files)
                    tohere=os.path.join(SASMODELS_DEST_MODELS,files)
                    shutil.copy(fromhere,tohere)

def retrieve_bumps_docs():
    """
    Copies select files from the bumps documentation into fitting perspective
    """
    if os.path.exists(BUMPS_DOCS):
        print "=== Retrieve BUMPS Docs ==="
        filenames = [os.path.join(BUMPS_DOCS, "optimizer.rst")]
        filenames += glob(os.path.join(BUMPS_DOCS, "dream-*.png"))
        filenames += glob(os.path.join(BUMPS_DOCS, "fit-*.png"))
        for f in filenames:
            print "Copying file", f
            shutil.copy(f, BUMPS_TARGET)
    else:
        print """
*** Error *** missing directory %s
The documentation will not include the optimizer selection section.
Checkout the bumps source tree and rebuild the docs.

""" % BUMPS_DOCS

def apidoc():
    """
    Runs sphinx-apidoc to generate .rst files from the docstrings in .py files
    in the SasView build directory.
    """
    print "=== Generate API Rest Files ==="

    # Clean directory before generating a new version.
    _remove_dir(SPHINX_SOURCE_API)

    subprocess.call(["sphinx-apidoc",
                     "-o", SPHINX_SOURCE_API, # Output dir.
                     "-d", "8", # Max depth of TOC.
                     SASVIEW_BUILD])

def build():
    """
    Runs sphinx-build.  Reads in all .rst files and spits out the final html.
    """
    print "=== Build HTML Docs from Rest Files ==="
    subprocess.call(["sphinx-build",
                     "-b", "html", # Builder name. TODO: accept as arg to setup.py.
                     "-d", os.path.join(SPHINX_BUILD, "doctrees"),
                     SPHINX_SOURCE,
                     os.path.join(SPHINX_BUILD, "html")])

    print "=== Copy HTML Docs to Build Directory ==="
    html = os.path.join(SPHINX_BUILD, "html")
    copy_tree(html, SASVIEW_DOCS)

def rebuild():
    clean()
    retrieve_user_docs()
    retrieve_bumps_docs()
    apidoc()
    build()

    print "=== Done ==="

if __name__ == "__main__":
    rebuild()
<commit_msg>Fix to docs build errors resulting from reorganisation of image file locations<commit_after>#!/usr/bin/env python
"""
Functions for building sphinx docs.

For more information on the invocation of sphinx see:
http://sphinx-doc.org/invocation.html
"""
import subprocess
import os
import sys
import fnmatch
import shutil
import imp
from glob import glob

from distutils.dir_util import copy_tree
from distutils.util import get_platform
from shutil import copy
from os import listdir

platform = '.%s-%s'%(get_platform(),sys.version[:3])

CURRENT_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))

run = imp.load_source('run', os.path.join(CURRENT_SCRIPT_DIR, '..', '..', 'run.py'))
run.prepare()

SASVIEW_SRC = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "src")
SASVIEW_BUILD = os.path.abspath(os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "build", "lib"+platform))
SASVIEW_DOCS = os.path.join(SASVIEW_BUILD, "doc")
SASVIEW_TEST = os.path.join(SASVIEW_SRC, "..", "sasview", "test", "media")

# Need to slurp in the new sasmodels model definitions to replace the old model_functions.rst
# We are currently here:
#/sasview-local-trunk/docs/sphinx-docs/build_sphinx.py
SASMODELS_SOURCE_PROLOG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc")
SASMODELS_SOURCE_REF_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "ref", "models")
SASMODELS_SOURCE_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "model")
SASMODELS_SOURCE_IMG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "model", "img")
SASMODELS_SOURCE_AUTOIMG = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..", "sasmodels", "doc", "_build", "html","_images")
SASMODELS_DEST_PROLOG = os.path.join(CURRENT_SCRIPT_DIR, "source")
SASMODELS_DEST_REF_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "source", "user")
SASMODELS_DEST_MODELS = os.path.join(CURRENT_SCRIPT_DIR, "source", "user", "models")
SASMODELS_DEST_IMG = os.path.join(CURRENT_SCRIPT_DIR,  "source", "user", "model-imgs", "new-models")
SASMODELS_DEST_BUILDIMG = os.path.join(CURRENT_SCRIPT_DIR,  "source", "user", "models", "img")

#if os.path.exists(SASMODELS_SOURCE_PROLOG):
#    print "Found models prolog folder at ", SASMODELS_SOURCE_PROLOG
#if os.path.exists(SASMODELS_SOURCE_REF_MODELS):
#    print "Found models ref folder at ", SASMODELS_SOURCE_REF_MODELS
#if os.path.exists(SASMODELS_SOURCE_MODELS):
#    print "Found models folder at ", SASMODELS_SOURCE_MODELS
#if os.path.exists(SASMODELS_SOURCE_IMG):
#    print "Found img folder at ", SASMODELS_SOURCE_IMG
#if os.path.exists(SASMODELS_DEST_REF_MODELS):
#    print "Found models ref folder at ", SASMODELS_DEST_REF_MODELS
#if os.path.exists(SASMODELS_DEST_MODELS):
#    print "Found models folder at ", SASMODELS_DEST_MODELS
#if os.path.exists(SASMODELS_DEST_IMG):
#    print "Found img folder at ", SASMODELS_DEST_IMG
#sys.exit()

SPHINX_BUILD = os.path.join(CURRENT_SCRIPT_DIR, "build")
SPHINX_SOURCE = os.path.join(CURRENT_SCRIPT_DIR, "source")
SPHINX_SOURCE_API = os.path.join(SPHINX_SOURCE, "dev", "api")
SPHINX_SOURCE_GUIFRAME = os.path.join(SPHINX_SOURCE, "user", "sasgui", "guiframe")
SPHINX_SOURCE_MODELS = os.path.join(SPHINX_SOURCE, "user", "models")
SPHINX_SOURCE_PERSPECTIVES = os.path.join(SPHINX_SOURCE, "user", "sasgui", "perspectives")
SPHINX_SOURCE_TEST = os.path.join(SPHINX_SOURCE, "test")

BUMPS_DOCS = os.path.join(CURRENT_SCRIPT_DIR, "..", "..", "..",
                          "bumps", "doc", "guide")
BUMPS_TARGET = os.path.join(SPHINX_SOURCE_PERSPECTIVES, "fitting")

def inplace_change(filename, old_string, new_string):
# Thanks to http://stackoverflow.com/questions/4128144/replace-string-within-file-contents
        s=open(filename).read()
        if old_string in s:
                print 'Changing "{old_string}" to "{new_string}"'.format(**locals())
                s=s.replace(old_string, new_string)
                f=open(filename, 'w')
                f.write(s)
                f.flush()
                f.close()
        else:
                print 'No occurences of "{old_string}" found.'.format(**locals())

def _remove_dir(dir_path):
    """Removes the given directory."""
    if os.path.isdir(dir_path):
        print "Removing \"%s\"... " % dir_path
        shutil.rmtree(dir_path)

def clean():
    """
    Clean the sphinx build directory.
    """
    print "=== Cleaning Sphinx Build ==="
    _remove_dir(SASVIEW_DOCS)
    _remove_dir(SPHINX_BUILD)
    _remove_dir(SPHINX_SOURCE_GUIFRAME)
    _remove_dir(SPHINX_SOURCE_MODELS)
    _remove_dir(SPHINX_SOURCE_PERSPECTIVES)
    _remove_dir(SPHINX_SOURCE_TEST)

def retrieve_user_docs():
    """
    Copies across the contents of any media/ directories in src/, and puts them
    in an appropriately named directory of docs/sphinx-docs/source/. For
    example:

        sas/../[MODULE]/media/dir/A.rst
        sas/../[MODULE]/media/B.rst

    gets copied to a new location:

        docs/sphinx-docs/source/user/[MODULE]/dir/A.rst
        docs/sphinx-docs/source/user/[MODULE]/B.rst

    so that Sphinx may pick it up when generating the documentation.
    """
    print "=== Retrieve User Docs ==="

    # Copy documentation files from their "source" to their "destination".
    for root, dirnames, _ in os.walk(SASVIEW_SRC):
        for dirname in fnmatch.filter(dirnames, 'media'):

            docs = os.path.abspath(os.path.join(root, dirname))
            print "Found docs folder at \"%s\"." % docs

            dest_dir_part = os.path.dirname(os.path.relpath(docs, SASVIEW_SRC))
            if os.sep in dest_dir_part:
                dest_dir_part = dest_dir_part[dest_dir_part.index(os.sep) + 1:]
            dest_dir = os.path.join(SPHINX_SOURCE, "user", dest_dir_part)

            copy_tree(docs, dest_dir)

    # Now pickup testdata_help.rst
#    print os.path.abspath(SASVIEW_TEST)
#    print os.path.abspath(SPHINX_SOURCE_TEST)
    print "=== Including Test Data Docs ==="
    if os.path.exists(SASVIEW_TEST):
       print "Found docs folder at ", SASVIEW_TEST
       shutil.copytree(SASVIEW_TEST, SPHINX_SOURCE_TEST)

    print "=== And the Sasmodels Docs ==="
    # Make sure we have the relevant images for the new sasmodels documentation
    # First(!) we'll make a local reference copy for SasView (/new-models will be cleaned each build)
    if os.path.exists(SASMODELS_SOURCE_IMG):
        print "Found img  folder SASMODELS_SOURCE_IMG    at ", SASMODELS_SOURCE_IMG
        if os.path.exists(SASMODELS_DEST_IMG):
            print "Found img  folder SASMODELS_DEST_IMG      at ", SASMODELS_DEST_IMG
            print "Copying sasmodels model image files..."
            for files in os.listdir(SASMODELS_SOURCE_IMG):
                fromhere=os.path.join(SASMODELS_SOURCE_IMG,files)
                tohere=os.path.join(SASMODELS_DEST_IMG,files)
                shutil.copy(fromhere,tohere)

    if os.path.exists(SASMODELS_SOURCE_AUTOIMG):
        print "Found img  folder SASMODELS_SOURCE_AUTOIMG    at ", SASMODELS_SOURCE_AUTOIMG
        if os.path.exists(SASMODELS_DEST_IMG):
            print "Copying sasmodels model auto-generated image files..."
            for files in os.listdir(SASMODELS_SOURCE_AUTOIMG):
                fromhere=os.path.join(SASMODELS_SOURCE_AUTOIMG,files)
                tohere=os.path.join(SASMODELS_DEST_IMG,files)
                shutil.copy(fromhere,tohere)

    # And the rst prolog with the unit substitutions
    if os.path.exists(SASMODELS_SOURCE_PROLOG):
        print "Found prolog folder SASMODELS_SOURCE_PROLOG at ", SASMODELS_SOURCE_PROLOG
        if os.path.exists(SASMODELS_DEST_PROLOG):
            print "Found docs folder SASMODELS_DEST_PROLOG   at ", SASMODELS_DEST_PROLOG
            print "Copying sasmodels rst_prolog file..."
            for files in os.listdir(SASMODELS_SOURCE_PROLOG):
                if files.startswith("rst"):
                    fromhere=os.path.join(SASMODELS_SOURCE_PROLOG,files)
                    tohere=os.path.join(SASMODELS_DEST_PROLOG,files)
                    shutil.copy(fromhere,tohere)

    if os.path.exists(SASMODELS_SOURCE_REF_MODELS):
        print "Found docs folder SASMODELS_SOURCE_REF_MODELS at ", SASMODELS_SOURCE_REF_MODELS
        if os.path.exists(SASMODELS_DEST_REF_MODELS):
            print "Found docs folder SASMODELS_DEST_REF_MODELS   at ", SASMODELS_DEST_REF_MODELS
            print "Copying sasmodels model toctree files..."
            for files in os.listdir(SASMODELS_SOURCE_REF_MODELS):
                if files.endswith(".rst"):
                    fromhere=os.path.join(SASMODELS_SOURCE_REF_MODELS,files)
                    tohere=os.path.join(SASMODELS_DEST_REF_MODELS,files)
                    shutil.copy(fromhere,tohere)
    # But need to change the path to the model docs in the tocs
    for files in os.listdir(SASMODELS_DEST_REF_MODELS):
#        print files
        if files.startswith("shape"):
            print "Changing toc paths in", files
            inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS,files), "../../model/", "models/")
        if files.startswith("sphere"):
            print "Changing toc paths in", files
            inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS,files), "../../model/", "models/")
        if files.startswith("custom"):
            print "Changing toc paths in", files
            inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS,files), "../../model/", "models/")
        if files.startswith("structure"):
            print "Changing toc paths in", files
            inplace_change(os.path.join(SASMODELS_DEST_REF_MODELS,files), "../../model/", "models/")

    if os.path.exists(SASMODELS_SOURCE_MODELS):
        print "Found docs folder SASMODELS_SOURCE_MODELS at ", SASMODELS_SOURCE_MODELS
        if os.path.exists(SASMODELS_DEST_MODELS):
            print "Found docs folder SASMODELS_DEST_MODELS   at ", SASMODELS_DEST_MODELS
            print "Copying sasmodels model files..."
            for files in os.listdir(SASMODELS_SOURCE_MODELS):
                if files.endswith(".rst"):
                    fromhere=os.path.join(SASMODELS_SOURCE_MODELS,files)
                    tohere=os.path.join(SASMODELS_DEST_MODELS,files)
                    shutil.copy(fromhere,tohere)
        else:
            print "Missing docs folder SASMODELS_DEST_MODELS at ", SASMODELS_DEST_MODELS
            os.makedirs(SASMODELS_DEST_MODELS)
            if not os.path.exists(SASMODELS_DEST_BUILDIMG):
                os.makedirs(SASMODELS_DEST_BUILDIMG)
            print "Created docs folder SASMODELS_DEST_MODELS at ", SASMODELS_DEST_MODELS
            print "Copying model files for build..."
            for files in os.listdir(SASMODELS_SOURCE_MODELS):
                if files.endswith(".rst"):
                    fromhere=os.path.join(SASMODELS_SOURCE_MODELS,files)
                    tohere=os.path.join(SASMODELS_DEST_MODELS,files)
                    shutil.copy(fromhere,tohere)
            # No choice but to do this because model files are all coded for images in /models/img
            print "Copying image files for build..."
            for files in os.listdir(SASMODELS_DEST_IMG):
                fromhere=os.path.join(SASMODELS_DEST_IMG,files)
                tohere=os.path.join(SASMODELS_DEST_BUILDIMG,files)
                shutil.copy(fromhere,tohere)

def retrieve_bumps_docs():
    """
    Copies select files from the bumps documentation into fitting perspective
    """
    if os.path.exists(BUMPS_DOCS):
        print "=== Retrieve BUMPS Docs ==="
        filenames = [os.path.join(BUMPS_DOCS, "optimizer.rst")]
        filenames += glob(os.path.join(BUMPS_DOCS, "dream-*.png"))
        filenames += glob(os.path.join(BUMPS_DOCS, "fit-*.png"))
        for f in filenames:
            print "Copying file", f
            shutil.copy(f, BUMPS_TARGET)
    else:
        print """
*** Error *** missing directory %s
The documentation will not include the optimizer selection section.
Checkout the bumps source tree and rebuild the docs.

""" % BUMPS_DOCS

def apidoc():
    """
    Runs sphinx-apidoc to generate .rst files from the docstrings in .py files
    in the SasView build directory.
    """
    print "=== Generate API Rest Files ==="

    # Clean directory before generating a new version.
    _remove_dir(SPHINX_SOURCE_API)

    subprocess.call(["sphinx-apidoc",
                     "-o", SPHINX_SOURCE_API, # Output dir.
                     "-d", "8", # Max depth of TOC.
                     SASVIEW_BUILD])

def build():
    """
    Runs sphinx-build.  Reads in all .rst files and spits out the final html.
    """
    print "=== Build HTML Docs from Rest Files ==="
    subprocess.call(["sphinx-build",
                     "-b", "html", # Builder name. TODO: accept as arg to setup.py.
                     "-d", os.path.join(SPHINX_BUILD, "doctrees"),
                     SPHINX_SOURCE,
                     os.path.join(SPHINX_BUILD, "html")])

    print "=== Copy HTML Docs to Build Directory ==="
    html = os.path.join(SPHINX_BUILD, "html")
    copy_tree(html, SASVIEW_DOCS)

def rebuild():
    clean()
    retrieve_user_docs()
    retrieve_bumps_docs()
    apidoc()
    build()

    print "=== Done ==="

if __name__ == "__main__":
    rebuild()
<|endoftext|>