<commit_before>import os
import time
import random
import hashlib
import getpass
import sys
import struct
import rsa
import multiprocessing
from Crypto.Cipher import AES
from shemutils.logger import Logger

class Key(object):
    """
    Class Key written by shemhazai
    This method generates a random key, unencoded or encoded in base64.
    """
    def __init__(self, bits):
        self.key_size = bits
        if self._parse_bits() != 0:
            self.key = None
        else:
            self.key = self._generate_key()

    def _parse_bits(self):
        """Parse if the input int is divisible by 2"""
        if self.key_size % 2 != 0:
            return -1
        return 0

    def _generate_key(self):
        """Generate random bytes and join them into a single string"""
        return ''.join(chr(random.randint(0, 0xFF)) for i in range(self.key_size))

    def get(self, encoded=False):
        """
        Method for returning the generated key
        :param encoded: Boolean
        :return: key string
        """
        if encoded is True:
            return self.key.encode("base64")

        return self.key

class Encryption:
    """This module uses pycrypto for AES encryption"""

    def __init__(self):
        pass

    @staticmethod
    def create_iv():
        return ''.join(chr(random.randint(0, 0xFF)) for i in range(16))

    @staticmethod
    def hash256(string):
        return hashlib.sha256(string).digest()

    @staticmethod
    def hash512(string):
        return hashlib.sha512(string).digest()

    @staticmethod
    def hashmd5(string):
        return hashlib.md5(string).digest()

    @staticmethod
    def get_key(bits=256):
        sys.stderr.write("Bit-size selected: %d\n" % bits)
        k = str()
        c = str("c")
        while k != c:
            if k == c:
                break
            k = getpass.getpass("Type your key: ", sys.stderr)
            c = getpass.getpass("Confirm your key: ", sys.stderr)
        if bits == 256:
            sys.stderr.write("Generating 256-bit key ...\n")
            return Encryption.hash256(k)
        elif bits == 128:
            sys.stderr.write("Generating 128-bit key ...\n")
            return Encryption.hashmd5(k)

    @staticmethod
    def encrypt_file(file_name, key, iv, output=None, chunksize=64*1024):
        if not output:
            dire_name = os.path.dirname(file_name)
            base_name = os.path.basename(file_name)
            if dire_name != "":
                output = dire_name + os.sep + base_name + ".enc"
            else:
                output = base_name + ".enc"
        filesize = os.path.getsize(file_name)
        encryptor = AES.new(key, AES.MODE_CBC, iv)
        with open(file_name, "rb") as infile:
            with open(output, "wb") as outfile:
                outfile.write(struct.pack("<Q", filesize))
                outfile.write(iv)
                while True:
                    chunk = infile.read(chunksize)
                    if len(chunk) == 0:
                        break
                    elif len(chunk) % 16 != 0:
                        chunk += " " * (16 - len(chunk) % 16)
                    outfile.write(encryptor.encrypt(chunk))
        return True

    @staticmethod
    def decrypt_file(file_name, key, output=None, chunksize=64*1024):
        if not output:
            root_name, ext = os.path.splitext(file_name)
            if ext != ".enc":
                return False
            dir_name = os.path.dirname(root_name) + os.sep
            f_name = os.path.basename(root_name)
            if dir_name != "/":
                output = dir_name + f_name
            else:
                output = f_name
        with open(file_name, "rb") as infile:
            origsize = struct.unpack("<Q", infile.read(struct.calcsize('Q')))[0]
            iv = infile.read(16)
            decryptor = AES.new(key, AES.MODE_CBC, iv)
            with open(output, "wb") as outfile:
                while True:
                    chunk = infile.read(chunksize)
                    if len(chunk) == 0:
                        break
                    outfile.write(decryptor.decrypt(chunk))
                outfile.truncate(origsize)
        return True

    @staticmethod
    def get_chunk(msg, index, chunksize=16):
        return msg[index:index+chunksize]

    @staticmethod
    def split_string(string, chunksize=16):
        output = []
        for x in range(0, len(string), chunksize):
            part = Encryption.get_chunk(string, x, chunksize=chunksize)
            if len(part) % 16 != 0:
                part += " " * (16 - len(part) % 16)
            output.append(part)
        return output

    @staticmethod
    def encrypt_message(plaintext, key, iv):
        psize = sys.getsizeof(plaintext)
        encryptor = AES.new(key, AES.MODE_CBC, iv)
        cipher = str()
        cipher += struct.pack("<Q", psize)
        cipher += iv
        for chunk in Encryption.split_string(plaintext):
            cipher += encryptor.encrypt(chunk)
        return cipher

    @staticmethod
    def decrypt_message(cipher, key):
        iv = cipher[struct.calcsize("Q"):struct.calcsize("3Q")]
        decryptor = AES.new(key, AES.MODE_CBC, iv)
        plaintext = str()
        for chunk in Encryption.split_string(cipher[struct.calcsize("3Q"):]):
            plaintext += decryptor.decrypt(chunk)
        return plaintext

class RSA:
    """This class have dependencies.
    multiprocessing, rsa modules are needed.
    """
    def __init__(self):
        self.logger = Logger("RSA")
        self.public_key = None
        self.private_key = None
        self.cpu_count = multiprocessing.cpu_count()

    def generate_keypair(self, bits=4096, v=True):
        if v is not False:
            self.logger.info("Generating new %d-bits key pair ..." % bits)
        t1 = time.time()
        self.public_key, self.private_key = rsa.newkeys(bits, poolsize=self.cpu_count)
        t2 = time.time()
        if v is not False:
            self.logger.info("Key pair generation took {0} seconds.".format(t2-t1))
        return True

    def encrypt_message(self, message, v=False):
        """

        :param message: string
        :param v: boolean  # stands for verbose
        :return:
        """
        if v is not False:
            self.logger.info("Encrypting message ...")
        s1 = time.time()
        crypto = rsa.encrypt(message, self.public_key)
        s2 = time.time()
        if v is not False:
            self.logger.info("Encryption success.")
            self.logger.info("Procedure took {0} seconds.".format(s2-s1))
        return crypto

    def decrypt_message(self, cipher, v=False):
        if v is not False:
            self.logger.info("Decrypting cipher ...")
        s1 = time.time()
        decrypto = rsa.decrypt(cipher, self.private_key)
        s2 = time.time()
        if v is not False:
            self.logger.info("Decryption success.")
            self.logger.info("Procedure took %d seconds." % (s2 - s1))
        return decrypto

    def save_keys(self, priv_f="private_key.pem",  pub_f="public_key.pem", v=False):
        if self.public_key is None:
            self.logger.error("Public Key does not exists. Generate it.")
            return False
        if self.private_key is None:
            self.logger.error("Private Key does not exists. Generate it.")
            return False
        with open(priv_f, "w") as priv:
            priv.write(self.private_key.save_pkcs1())
            self.logger.info("Private key saved to file '%s'" % priv_f)
        with open(pub_f, "w") as pub:
            pub.write(self.public_key.save_pkcs1())
            if v is True:
                self.logger.info("Public key saved to file '%s" % pub_f)
        return True

    def load_keys(self, priv_f, pub_f, v=False):
        if not os.path.isfile(priv_f):
            self.logger.error("Private key file does not exists.")
        if not os.path.isfile(pub_f):
            self.logger.error("Public key file does not exists.")
        with open(priv_f, "rb") as priv:
            priv_data = priv.read()
            priv_data = priv_data.encode("ascii")
        with open(pub_f, "rb") as pub:
            pub_data = pub.read()
            pub_data = pub_data.encode("ascii")
        self.private_key = rsa.PrivateKey.load_pkcs1(priv_data)
        self.public_key = rsa.PublicKey.load_pkcs1(pub_data)
        if v is True:
            self.logger.info("Key pair successfully loaded.")
        return True

<commit_msg>Porting code to py3.6<commit_after>import os
import time
import random
import hashlib
import getpass
import sys
import struct
import rsa
import multiprocessing
from Crypto.Cipher import AES
from shemutils.logger import Logger

class Key(object):
    """
    Class Key written by shemhazai
    This method generates a random key, unencoded or encoded in base64.
    """
    def __init__(self, bits):
        self.key_size = bits
        if self._parse_bits() != 0:
            self.key = None
        else:
            self.key = self._generate_key()

    def _parse_bits(self):
        """Parse if the input int is divisible by 2"""
        if self.key_size % 2 != 0:
            return -1
        return 0

    def _generate_key(self):
        """Generate random bytes and join them into a single string"""
        return ''.join(chr(random.randint(0, 0xFF)) for i in range(self.key_size))

    def get(self, encoded=False):
        """
        Method for returning the generated key
        :param encoded: Boolean
        :return: key string
        """
        if encoded is True:
            return self.key.encode("base64")

        return self.key

class Encryption:
    """This module uses pycrypto for AES encryption"""

    def __init__(self):
        pass

    @staticmethod
    def create_iv():
        return ''.join(chr(random.randint(0, 0xFF)) for _ in range(16))

    @staticmethod
    def hash256(string):
        return hashlib.sha256(string.encode()).digest()

    @staticmethod
    def hash512(string):
        return hashlib.sha512(string.encode()).digest()

    @staticmethod
    def hashmd5(string):
        return hashlib.md5(string.encode()).digest()

    @staticmethod
    def get_key(bits=256):
        sys.stderr.write("Bit-size selected: %d\n" % bits)
        k = str()
        c = str("c")
        while k != c:
            if k == c:
                break
            k = getpass.getpass("Type your key: ", sys.stderr)
            c = getpass.getpass("Confirm your key: ", sys.stderr)
        if bits == 256:
            sys.stderr.write("Generating 256-bit key ...\n")
            return Encryption.hash256(k)
        elif bits == 128:
            sys.stderr.write("Generating 128-bit key ...\n")
            return Encryption.hashmd5(k)

    @staticmethod
    def encrypt_file(file_name, key, iv, output=None, chunksize=64*1024):
        if not output:
            dire_name = os.path.dirname(file_name)
            base_name = os.path.basename(file_name)
            if dire_name != "":
                output = dire_name + os.sep + base_name + ".enc"
            else:
                output = base_name + ".enc"
        filesize = os.path.getsize(file_name)
        encryptor = AES.new(key, AES.MODE_CBC, iv)
        with open(file_name, "rb") as infile:
            with open(output, "wb") as outfile:
                outfile.write(struct.pack("<Q", filesize))
                outfile.write(iv)
                while True:
                    chunk = infile.read(chunksize)
                    if len(chunk) == 0:
                        break
                    elif len(chunk) % 16 != 0:
                        chunk += " " * (16 - len(chunk) % 16)
                    outfile.write(encryptor.encrypt(chunk))
        return True

    @staticmethod
    def decrypt_file(file_name, key, output=None, chunksize=64*1024):
        if not output:
            root_name, ext = os.path.splitext(file_name)
            if ext != ".enc":
                return False
            dir_name = os.path.dirname(root_name) + os.sep
            f_name = os.path.basename(root_name)
            if dir_name != "/":
                output = dir_name + f_name
            else:
                output = f_name
        with open(file_name, "rb") as infile:
            origsize = struct.unpack("<Q", infile.read(struct.calcsize('Q')))[0]
            iv = infile.read(16)
            decryptor = AES.new(key, AES.MODE_CBC, iv)
            with open(output, "wb") as outfile:
                while True:
                    chunk = infile.read(chunksize)
                    if len(chunk) == 0:
                        break
                    outfile.write(decryptor.decrypt(chunk))
                outfile.truncate(origsize)
        return True

    @staticmethod
    def get_chunk(msg, index, chunksize=16):
        return msg[index:index+chunksize]

    @staticmethod
    def split_string(string, chunksize=16):
        output = []
        for x in range(0, len(string), chunksize):
            part = Encryption.get_chunk(string, x, chunksize=chunksize)
            if len(part) % 16 != 0:
                part += " " * (16 - len(part) % 16)
            output.append(part)
        return output

    @staticmethod
    def encrypt_message(plaintext, key, iv):
        psize = sys.getsizeof(plaintext)
        encryptor = AES.new(key, AES.MODE_CBC, iv)
        cipher = str()
        cipher += struct.pack("<Q", psize)
        cipher += iv
        for chunk in Encryption.split_string(plaintext):
            cipher += encryptor.encrypt(chunk)
        return cipher

    @staticmethod
    def decrypt_message(cipher, key):
        iv = cipher[struct.calcsize("Q"):struct.calcsize("3Q")]
        decryptor = AES.new(key, AES.MODE_CBC, iv)
        plaintext = str()
        for chunk in Encryption.split_string(cipher[struct.calcsize("3Q"):]):
            plaintext += decryptor.decrypt(chunk)
        return plaintext

class RSA:
    """This class have dependencies.
    multiprocessing, rsa modules are needed.
    """
    def __init__(self):
        self.logger = Logger("RSA")
        self.public_key = None
        self.private_key = None
        self.cpu_count = multiprocessing.cpu_count()

    def generate_keypair(self, bits=4096, v=True):
        if v is not False:
            self.logger.info("Generating new %d-bits key pair ..." % bits)
        t1 = time.time()
        self.public_key, self.private_key = rsa.newkeys(bits, poolsize=self.cpu_count)
        t2 = time.time()
        if v is not False:
            self.logger.info("Key pair generation took {0} seconds.".format(t2-t1))
        return True

    def encrypt_message(self, message, v=False):
        """

        :param message: string
        :param v: boolean  # stands for verbose
        :return:
        """
        if v is not False:
            self.logger.info("Encrypting message ...")
        s1 = time.time()
        crypto = rsa.encrypt(message, self.public_key)
        s2 = time.time()
        if v is not False:
            self.logger.info("Encryption success.")
            self.logger.info("Procedure took {0} seconds.".format(s2-s1))
        return crypto

    def decrypt_message(self, cipher, v=False):
        if v is not False:
            self.logger.info("Decrypting cipher ...")
        s1 = time.time()
        decrypto = rsa.decrypt(cipher, self.private_key)
        s2 = time.time()
        if v is not False:
            self.logger.info("Decryption success.")
            self.logger.info("Procedure took %d seconds." % (s2 - s1))
        return decrypto

    def save_keys(self, priv_f="private_key.pem",  pub_f="public_key.pem", v=False):
        if self.public_key is None:
            self.logger.error("Public Key does not exists. Generate it.")
            return False
        if self.private_key is None:
            self.logger.error("Private Key does not exists. Generate it.")
            return False
        with open(priv_f, "w") as priv:
            priv.write(self.private_key.save_pkcs1())
            self.logger.info("Private key saved to file '%s'" % priv_f)
        with open(pub_f, "w") as pub:
            pub.write(self.public_key.save_pkcs1())
            if v is True:
                self.logger.info("Public key saved to file '%s" % pub_f)
        return True

    def load_keys(self, priv_f, pub_f, v=False):
        if not os.path.isfile(priv_f):
            self.logger.error("Private key file does not exists.")
        if not os.path.isfile(pub_f):
            self.logger.error("Public key file does not exists.")
        with open(priv_f, "rb") as priv:
            priv_data = priv.read()
            priv_data = priv_data.encode("ascii")
        with open(pub_f, "rb") as pub:
            pub_data = pub.read()
            pub_data = pub_data.encode("ascii")
        self.private_key = rsa.PrivateKey.load_pkcs1(priv_data)
        self.public_key = rsa.PublicKey.load_pkcs1(pub_data)
        if v is True:
            self.logger.info("Key pair successfully loaded.")
        return True

<|endoftext|>


<commit_before>import optparse
from faas import app

default_host = '127.0.0.1'
default_port = '5000'
parser = optparse.OptionParser()
parser.add_option('-H', '--host',
                  help='The hostname of the app. [default %s]' % default_host,
                  default=default_host)
parser.add_option('-P', '--port',
                  help='Port for the app. [default %s]' % default_port,
                  default=default_port)
parser.add_option('-d', '--debug', action='store_true', dest='debug')
options, _ = parser.parse_args()

app.run(debug=options.debug,
        host=options.host,
        port=int(options.port))

<commit_msg>Accept outside connections<commit_after>import optparse
from faas import app

default_host = '0.0.0.0'
default_port = '5000'
parser = optparse.OptionParser()
parser.add_option('-H', '--host',
                  help='The hostname of the app. [default %s]' % default_host,
                  default=default_host)
parser.add_option('-P', '--port',
                  help='Port for the app. [default %s]' % default_port,
                  default=default_port)
parser.add_option('-d', '--debug', action='store_true', dest='debug')
options, _ = parser.parse_args()

app.run(debug=options.debug,
        host=options.host,
        port=int(options.port))

<|endoftext|>


<commit_before># pylint:disable=missing-docstring

import datetime
import json
import uuid

import factory
from factory.fuzzy import FuzzyText
import pytz
from django.contrib.auth.models import User  # lint-amnesty, pylint: disable=imported-auth-user

from openedx.core.djangoapps.credit.models import (
    CreditCourse,
    CreditEligibility,
    CreditProvider,
    CreditRequest,
    CreditRequirement,
    CreditRequirementStatus
)
from common.djangoapps.util.date_utils import to_timestamp

class CreditCourseFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditCourse

    course_key = FuzzyText(prefix='fake.org/', suffix='/fake.run')
    enabled = True

class CreditRequirementFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditRequirement

    course = factory.SubFactory(CreditCourseFactory)

class CreditRequirementStatusFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditRequirementStatus

    requirement = factory.SubFactory(CreditRequirementFactory)
    status = CreditRequirementStatus.REQUIREMENT_STATUS_CHOICES[0][0]

class CreditProviderFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditProvider

    provider_id = FuzzyText(length=5)
    provider_url = FuzzyText(prefix='http://')

class CreditEligibilityFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditEligibility

    course = factory.SubFactory(CreditCourseFactory)

class CreditRequestFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditRequest

    uuid = factory.LazyAttribute(lambda o: uuid.uuid4().hex)  # pylint: disable=undefined-variable

    # pylint: disable=access-member-before-definition,attribute-defined-outside-init,no-self-argument,unused-argument
    @factory.post_generation
    def post(obj, create, extracted, **kwargs):
        """
        Post-generation handler.

        Sets up parameters field.
        """
        if not obj.parameters:
            course_key = obj.course.course_key
            user = User.objects.get(username=obj.username)
            user_profile = user.profile

            obj.parameters = json.dumps({
                "request_uuid": obj.uuid,
                "timestamp": to_timestamp(datetime.datetime.now(pytz.UTC)),
                "course_org": course_key.org,
                "course_num": course_key.course,
                "course_run": course_key.run,
                "final_grade": '0.96',
                "user_username": user.username,
                "user_email": user.email,
                "user_full_name": user_profile.name,
                "user_mailing_address": "",
                "user_country": user_profile.country.code or "",
            })

        obj.save()
<commit_msg>refactor: upgrading factory-boy.<commit_after># pylint:disable=missing-docstring

import datetime
import json
import uuid

import factory
from factory.fuzzy import FuzzyText
import pytz
from django.contrib.auth.models import User  # lint-amnesty, pylint: disable=imported-auth-user

from openedx.core.djangoapps.credit.models import (
    CreditCourse,
    CreditEligibility,
    CreditProvider,
    CreditRequest,
    CreditRequirement,
    CreditRequirementStatus
)
from common.djangoapps.util.date_utils import to_timestamp

class CreditCourseFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditCourse

    course_key = FuzzyText(prefix='fake.org/', suffix='/fake.run')
    enabled = True

class CreditRequirementFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditRequirement

    course = factory.SubFactory(CreditCourseFactory)

class CreditRequirementStatusFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditRequirementStatus

    requirement = factory.SubFactory(CreditRequirementFactory)
    status = CreditRequirementStatus.REQUIREMENT_STATUS_CHOICES[0][0]

class CreditProviderFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditProvider

    provider_id = FuzzyText(length=5)
    provider_url = FuzzyText(prefix='http://')

class CreditEligibilityFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditEligibility

    course = factory.SubFactory(CreditCourseFactory)

class CreditRequestFactory(factory.DjangoModelFactory):
    class Meta:
        model = CreditRequest

    uuid = factory.LazyAttribute(lambda o: uuid.uuid4().hex)  # pylint: disable=undefined-variable

    # pylint: disable=access-member-before-definition,attribute-defined-outside-init,no-self-argument,unused-argument
    @factory.post_generation
    def post(obj, create, extracted, **kwargs):
        """
        Post-generation handler.

        Sets up parameters field.
        """
        if not obj.parameters:
            course_key = obj.course.course_key
            user = User.objects.get(username=obj.username)
            user_profile = user.profile

            obj.parameters = json.dumps({
                "request_uuid": obj.uuid,
                "timestamp": to_timestamp(datetime.datetime.now(pytz.UTC)),
                "course_org": course_key.org,
                "course_num": course_key.course,
                "course_run": course_key.run,
                "final_grade": '0.96',
                "user_username": user.username,
                "user_email": user.email,
                "user_full_name": user_profile.name,
                "user_mailing_address": "",
                "user_country": user_profile.country.code or "",
            })

        obj.save()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Install software from the FreeBSD ``ports(7)`` system

.. versionadded:: 2014.1.0

This module allows you to install ports using ``BATCH=yes`` to bypass
configuration prompts. It is recommended to use the :mod:`ports state
<salt.states.freebsdports>` to install ports, but it it also possible to use
this module exclusively from the command line.

.. code-block:: bash

    salt minion-id ports.config security/nmap IPV6=off
    salt minion-id ports.install security/nmap
'''
from __future__ import absolute_import

# Import python libs
import fnmatch
import os
import re
import logging

# Import salt libs
import salt.utils
from six import string_types
from salt.exceptions import SaltInvocationError, CommandExecutionError
import salt.utils.six as six

log = logging.getLogger(__name__)

# Define the module's virtual name
__virtualname__ = 'ports'

def __virtual__():
    return __virtualname__ if __grains__.get('os', '') == 'FreeBSD' else False

def _portsnap():
    '''
    Return 'portsnap --interactive' for FreeBSD 10, otherwise 'portsnap'
    '''
    return 'portsnap{0}'.format(
        ' --interactive' if float(__grains__['osrelease']) >= 10
        else ''
    )

def _check_portname(name):
    '''
    Check if portname is valid and whether or not the directory exists in the
    ports tree.
    '''
    if not isinstance(name, string_types) or '/' not in name:
        raise SaltInvocationError(
            'Invalid port name {0!r} (category required)'.format(name)
        )

    path = os.path.join('/usr/ports', name)
    if not os.path.isdir(path):
        raise SaltInvocationError('Path {0!r} does not exist'.format(path))

    return path

def _options_dir(name):
    '''
    Retrieve the path to the dir containing OPTIONS file for a given port
    '''
    _check_portname(name)
    _root = '/var/db/ports'

    # New path: /var/db/ports/category_portname
    new_dir = os.path.join(_root, name.replace('/', '_'))
    # Old path: /var/db/ports/portname
    old_dir = os.path.join(_root, name.split('/')[-1])

    if os.path.isdir(old_dir):
        return old_dir
    return new_dir

def _options_file_exists(name):
    '''
    Returns True/False based on whether or not the options file for the
    specified port exists.
    '''
    return os.path.isfile(os.path.join(_options_dir(name), 'options'))

def _write_options(name, configuration):
    '''
    Writes a new OPTIONS file
    '''
    _check_portname(name)

    pkg = next(iter(configuration))
    conf_ptr = configuration[pkg]

    dirname = _options_dir(name)
    if not os.path.isdir(dirname):
        try:
            os.makedirs(dirname)
        except OSError as exc:
            raise CommandExecutionError(
                'Unable to make {0}: {1}'.format(dirname, exc)
            )

    with salt.utils.fopen(os.path.join(dirname, 'options'), 'w') as fp_:
        sorted_options = list(conf_ptr.keys())
        sorted_options.sort()
        fp_.write(
            '# This file was auto-generated by Salt (http://saltstack.com)\n'
            '# Options for {0}\n'
            '_OPTIONS_READ={0}\n'
            '_FILE_COMPLETE_OPTIONS_LIST={1}\n'
            .format(pkg, ' '.join(sorted_options))
        )
        opt_tmpl = 'OPTIONS_FILE_{0}SET+={1}\n'
        for opt in sorted_options:
            fp_.write(
                opt_tmpl.format(
                    '' if conf_ptr[opt] == 'on' else 'UN',
                    opt
                )
            )

def _normalize(val):
    '''
    Fix Salt's yaml-ification of on/off, and otherwise normalize the on/off
    values to be used in writing the options file
    '''
    if isinstance(val, bool):
        return 'on' if val else 'off'
    return str(val).lower()

def install(name, clean=True):
    '''
    Install a port from the ports tree. Installs using ``BATCH=yes`` for
    non-interactive building. To set config options for a given port, use
    :mod:`ports.config <salt.modules.freebsdports.config>`.

    clean : True
        If ``True``, cleans after installation. Equivalent to running ``make
        install clean BATCH=yes``.

    .. note::

        It may be helpful to run this function using the ``-t`` option to set a
        higher timeout, since compiling a port may cause the Salt command to
        exceed the default timeout.

    CLI Example:

    .. code-block:: bash

        salt -t 1200 '*' ports.install security/nmap
    '''
    portpath = _check_portname(name)
    old = __salt__['pkg.list_pkgs']()
    if old.get(name.rsplit('/')[-1]):
        deinstall(name)
    result = __salt__['cmd.run_all'](
        'make install{0} BATCH=yes'.format(' clean' if clean else ''),
        cwd=portpath, reset_system_locale=False
    )
    if result['retcode'] != 0:
        __context__['ports.install_error'] = result['stderr']
    __context__.pop('pkg.list_pkgs', None)
    new = __salt__['pkg.list_pkgs']()
    ret = salt.utils.compare_dicts(old, new)
    if not ret and result['retcode'] == 0:
        # No change in package list, but the make install was successful.
        # Assume that the installation was a recompile with new options, and
        # set return dict so that changes are detected by the ports.installed
        # state.
        ret = {name: {'old': old.get(name, ''),
                      'new': new.get(name, '')}}
    return ret

def deinstall(name):
    '''
    De-install a port.

    CLI Example:

    .. code-block:: bash

        salt '*' ports.deinstall security/nmap
    '''
    portpath = _check_portname(name)
    old = __salt__['pkg.list_pkgs']()
    __salt__['cmd.run']('make deinstall BATCH=yes', cwd=portpath)
    __context__.pop('pkg.list_pkgs', None)
    new = __salt__['pkg.list_pkgs']()
    return salt.utils.compare_dicts(old, new)

def rmconfig(name):
    '''
    Clear the cached options for the specified port; run a ``make rmconfig``

    name
        The name of the port to clear

    CLI Example:

    .. code-block:: bash

        salt '*' ports.rmconfig security/nmap
    '''
    portpath = _check_portname(name)
    return __salt__['cmd.run']('make rmconfig', cwd=portpath)

def showconfig(name, default=False, dict_return=False):
    '''
    Show the configuration options for a given port.

    default : False
        Show the default options for a port (not necessarily the same as the
        current configuration)

    dict_return : False
        Instead of returning the output of ``make showconfig``, return the data
        in an dictionary

    CLI Example:

    .. code-block:: bash

        salt '*' ports.showconfig security/nmap
        salt '*' ports.showconfig security/nmap default=True
    '''
    portpath = _check_portname(name)

    if default and _options_file_exists(name):
        saved_config = showconfig(name, default=False, dict_return=True)
        rmconfig(name)
        if _options_file_exists(name):
            raise CommandExecutionError('Unable to get default configuration')
        default_config = showconfig(name, default=False,
                                    dict_return=dict_return)
        _write_options(name, saved_config)
        return default_config

    try:
        result = __salt__['cmd.run_all']('make showconfig', cwd=portpath)
        output = result['stdout'].splitlines()
        if result['retcode'] != 0:
            error = result['stderr']
        else:
            error = ''
    except TypeError:
        error = result

    if error:
        msg = ('Error running \'make showconfig\' for {0}: {1}'
               .format(name, error))
        log.error(msg)
        raise SaltInvocationError(msg)

    if not dict_return:
        return '\n'.join(output)

    if (not output) or ('configuration options' not in output[0]):
        return {}

    try:
        pkg = output[0].split()[-1].rstrip(':')
    except (IndexError, AttributeError, TypeError) as exc:
        log.error(
            'Unable to get pkg-version string: {0}'.format(exc)
        )
        return {}

    ret = {pkg: {}}
    output = output[1:]
    for line in output:
        try:
            opt, val, desc = re.match(
                r'\s+([^=]+)=(off|on): (.+)', line
            ).groups()
        except AttributeError:
            continue
        ret[pkg][opt] = val

    if not ret[pkg]:
        return {}
    return ret

def config(name, reset=False, **kwargs):
    '''
    Modify configuration options for a given port. Multiple options can be
    specified. To see the available options for a port, use
    :mod:`ports.showconfig <salt.modules.freebsdports.showconfig>`.

    name
        The port name, in ``category/name`` format

    reset : False
        If ``True``, runs a ``make rmconfig`` for the port, clearing its
        configuration before setting the desired options

    CLI Examples:

    .. code-block:: bash

        salt '*' ports.config security/nmap IPV6=off
    '''
    portpath = _check_portname(name)

    if reset:
        rmconfig(name)

    configuration = showconfig(name, dict_return=True)

    if not configuration:
        raise CommandExecutionError(
            'Unable to get port configuration for {0!r}'.format(name)
        )

    # Get top-level key for later reference
    pkg = next(iter(configuration))
    conf_ptr = configuration[pkg]

    opts = dict(
        (str(x), _normalize(kwargs[x]))
        for x in kwargs
        if not x.startswith('_')
    )

    bad_opts = [x for x in opts if x not in conf_ptr]
    if bad_opts:
        raise SaltInvocationError(
            'The following opts are not valid for port {0}: {1}'
            .format(name, ', '.join(bad_opts))
        )

    bad_vals = [
        '{0}={1}'.format(x, y) for x, y in six.iteritems(opts)
        if y not in ('on', 'off')
    ]
    if bad_vals:
        raise SaltInvocationError(
            'The following key/value pairs are invalid: {0}'
            .format(', '.join(bad_vals))
        )

    conf_ptr.update(opts)
    _write_options(name, configuration)

    new_config = showconfig(name, dict_return=True)
    try:
        new_config = new_config[next(iter(new_config))]
    except (StopIteration, TypeError):
        return False

    return all(conf_ptr[x] == new_config.get(x) for x in conf_ptr)

def update(extract=False):
    '''
    Update the ports tree

    extract : False
        If ``True``, runs a ``portsnap extract`` after fetching, should be used
        for first-time installation of the ports tree.

    CLI Example:

    .. code-block:: bash

        salt '*' ports.update
    '''
    result = __salt__['cmd.run_all']('{0} fetch'.format(_portsnap()))
    if not result['retcode'] == 0:
        raise CommandExecutionError(
            'Unable to fetch ports snapshot: {0}'.format(result['stderr'])
        )

    ret = []
    try:
        patch_count = re.search(
            r'Fetching (\d+) patches', result['stdout']
        ).group(1)
    except AttributeError:
        patch_count = 0

    try:
        new_port_count = re.search(
            r'Fetching (\d+) new ports or files', result['stdout']
        ).group(1)
    except AttributeError:
        new_port_count = 0

    ret.append('Applied {0} new patches'.format(patch_count))
    ret.append('Fetched {0} new ports or files'.format(new_port_count))

    if extract:
        result = __salt__['cmd.run_all']('{0} extract'.format(_portsnap()))
        if not result['retcode'] == 0:
            raise CommandExecutionError(
                'Unable to extract ports snapshot {0}'.format(result['stderr'])
            )

    result = __salt__['cmd.run_all']('{0} update'.format(_portsnap()))
    if not result['retcode'] == 0:
        raise CommandExecutionError(
            'Unable to apply ports snapshot: {0}'.format(result['stderr'])
        )

    __context__.pop('ports.list_all', None)
    return '\n'.join(ret)

def list_all():
    '''
    Lists all ports available.

    CLI Example:

    .. code-block:: bash

        salt '*' ports.list_all

    .. warning::

        Takes a while to run, and returns a **LOT** of output
    '''
    if 'ports.list_all' not in __context__:
        __context__['ports.list_all'] = []
        for path, dirs, files in os.walk('/usr/ports'):
            stripped = path[len('/usr/ports'):]
            if stripped.count('/') != 2 or stripped.endswith('/CVS'):
                continue
            __context__['ports.list_all'].append(stripped[1:])
    return __context__['ports.list_all']

def search(name):
    '''
    Search for matches in the ports tree. Globs are supported, and the category
    is optional

    CLI Examples:

    .. code-block:: bash

        salt '*' ports.search 'security/*'
        salt '*' ports.search 'security/n*'
        salt '*' ports.search nmap

    .. warning::

        Takes a while to run
    '''
    name = str(name)
    all_ports = list_all()
    if '/' in name:
        if name.count('/') > 1:
            raise SaltInvocationError(
                'Invalid search string {0!r}. Port names cannot have more '
                'than one slash'
            )
        else:
            return fnmatch.filter(all_ports, name)
    else:
        ret = []
        for port in all_ports:
            if fnmatch.fnmatch(port.rsplit('/')[-1], name):
                ret.append(port)
        return ret
<commit_msg>Replaced module six in file /salt/modules/freebsdports.py<commit_after># -*- coding: utf-8 -*-
'''
Install software from the FreeBSD ``ports(7)`` system

.. versionadded:: 2014.1.0

This module allows you to install ports using ``BATCH=yes`` to bypass
configuration prompts. It is recommended to use the :mod:`ports state
<salt.states.freebsdports>` to install ports, but it it also possible to use
this module exclusively from the command line.

.. code-block:: bash

    salt minion-id ports.config security/nmap IPV6=off
    salt minion-id ports.install security/nmap
'''
from __future__ import absolute_import

# Import python libs
import fnmatch
import os
import re
import logging

# Import salt libs
import salt.utils
from salt.utils.six import string_types
from salt.exceptions import SaltInvocationError, CommandExecutionError
import salt.utils.six as six

log = logging.getLogger(__name__)

# Define the module's virtual name
__virtualname__ = 'ports'

def __virtual__():
    return __virtualname__ if __grains__.get('os', '') == 'FreeBSD' else False

def _portsnap():
    '''
    Return 'portsnap --interactive' for FreeBSD 10, otherwise 'portsnap'
    '''
    return 'portsnap{0}'.format(
        ' --interactive' if float(__grains__['osrelease']) >= 10
        else ''
    )

def _check_portname(name):
    '''
    Check if portname is valid and whether or not the directory exists in the
    ports tree.
    '''
    if not isinstance(name, string_types) or '/' not in name:
        raise SaltInvocationError(
            'Invalid port name {0!r} (category required)'.format(name)
        )

    path = os.path.join('/usr/ports', name)
    if not os.path.isdir(path):
        raise SaltInvocationError('Path {0!r} does not exist'.format(path))

    return path

def _options_dir(name):
    '''
    Retrieve the path to the dir containing OPTIONS file for a given port
    '''
    _check_portname(name)
    _root = '/var/db/ports'

    # New path: /var/db/ports/category_portname
    new_dir = os.path.join(_root, name.replace('/', '_'))
    # Old path: /var/db/ports/portname
    old_dir = os.path.join(_root, name.split('/')[-1])

    if os.path.isdir(old_dir):
        return old_dir
    return new_dir

def _options_file_exists(name):
    '''
    Returns True/False based on whether or not the options file for the
    specified port exists.
    '''
    return os.path.isfile(os.path.join(_options_dir(name), 'options'))

def _write_options(name, configuration):
    '''
    Writes a new OPTIONS file
    '''
    _check_portname(name)

    pkg = next(iter(configuration))
    conf_ptr = configuration[pkg]

    dirname = _options_dir(name)
    if not os.path.isdir(dirname):
        try:
            os.makedirs(dirname)
        except OSError as exc:
            raise CommandExecutionError(
                'Unable to make {0}: {1}'.format(dirname, exc)
            )

    with salt.utils.fopen(os.path.join(dirname, 'options'), 'w') as fp_:
        sorted_options = list(conf_ptr.keys())
        sorted_options.sort()
        fp_.write(
            '# This file was auto-generated by Salt (http://saltstack.com)\n'
            '# Options for {0}\n'
            '_OPTIONS_READ={0}\n'
            '_FILE_COMPLETE_OPTIONS_LIST={1}\n'
            .format(pkg, ' '.join(sorted_options))
        )
        opt_tmpl = 'OPTIONS_FILE_{0}SET+={1}\n'
        for opt in sorted_options:
            fp_.write(
                opt_tmpl.format(
                    '' if conf_ptr[opt] == 'on' else 'UN',
                    opt
                )
            )

def _normalize(val):
    '''
    Fix Salt's yaml-ification of on/off, and otherwise normalize the on/off
    values to be used in writing the options file
    '''
    if isinstance(val, bool):
        return 'on' if val else 'off'
    return str(val).lower()

def install(name, clean=True):
    '''
    Install a port from the ports tree. Installs using ``BATCH=yes`` for
    non-interactive building. To set config options for a given port, use
    :mod:`ports.config <salt.modules.freebsdports.config>`.

    clean : True
        If ``True``, cleans after installation. Equivalent to running ``make
        install clean BATCH=yes``.

    .. note::

        It may be helpful to run this function using the ``-t`` option to set a
        higher timeout, since compiling a port may cause the Salt command to
        exceed the default timeout.

    CLI Example:

    .. code-block:: bash

        salt -t 1200 '*' ports.install security/nmap
    '''
    portpath = _check_portname(name)
    old = __salt__['pkg.list_pkgs']()
    if old.get(name.rsplit('/')[-1]):
        deinstall(name)
    result = __salt__['cmd.run_all'](
        'make install{0} BATCH=yes'.format(' clean' if clean else ''),
        cwd=portpath, reset_system_locale=False
    )
    if result['retcode'] != 0:
        __context__['ports.install_error'] = result['stderr']
    __context__.pop('pkg.list_pkgs', None)
    new = __salt__['pkg.list_pkgs']()
    ret = salt.utils.compare_dicts(old, new)
    if not ret and result['retcode'] == 0:
        # No change in package list, but the make install was successful.
        # Assume that the installation was a recompile with new options, and
        # set return dict so that changes are detected by the ports.installed
        # state.
        ret = {name: {'old': old.get(name, ''),
                      'new': new.get(name, '')}}
    return ret

def deinstall(name):
    '''
    De-install a port.

    CLI Example:

    .. code-block:: bash

        salt '*' ports.deinstall security/nmap
    '''
    portpath = _check_portname(name)
    old = __salt__['pkg.list_pkgs']()
    __salt__['cmd.run']('make deinstall BATCH=yes', cwd=portpath)
    __context__.pop('pkg.list_pkgs', None)
    new = __salt__['pkg.list_pkgs']()
    return salt.utils.compare_dicts(old, new)

def rmconfig(name):
    '''
    Clear the cached options for the specified port; run a ``make rmconfig``

    name
        The name of the port to clear

    CLI Example:

    .. code-block:: bash

        salt '*' ports.rmconfig security/nmap
    '''
    portpath = _check_portname(name)
    return __salt__['cmd.run']('make rmconfig', cwd=portpath)

def showconfig(name, default=False, dict_return=False):
    '''
    Show the configuration options for a given port.

    default : False
        Show the default options for a port (not necessarily the same as the
        current configuration)

    dict_return : False
        Instead of returning the output of ``make showconfig``, return the data
        in an dictionary

    CLI Example:

    .. code-block:: bash

        salt '*' ports.showconfig security/nmap
        salt '*' ports.showconfig security/nmap default=True
    '''
    portpath = _check_portname(name)

    if default and _options_file_exists(name):
        saved_config = showconfig(name, default=False, dict_return=True)
        rmconfig(name)
        if _options_file_exists(name):
            raise CommandExecutionError('Unable to get default configuration')
        default_config = showconfig(name, default=False,
                                    dict_return=dict_return)
        _write_options(name, saved_config)
        return default_config

    try:
        result = __salt__['cmd.run_all']('make showconfig', cwd=portpath)
        output = result['stdout'].splitlines()
        if result['retcode'] != 0:
            error = result['stderr']
        else:
            error = ''
    except TypeError:
        error = result

    if error:
        msg = ('Error running \'make showconfig\' for {0}: {1}'
               .format(name, error))
        log.error(msg)
        raise SaltInvocationError(msg)

    if not dict_return:
        return '\n'.join(output)

    if (not output) or ('configuration options' not in output[0]):
        return {}

    try:
        pkg = output[0].split()[-1].rstrip(':')
    except (IndexError, AttributeError, TypeError) as exc:
        log.error(
            'Unable to get pkg-version string: {0}'.format(exc)
        )
        return {}

    ret = {pkg: {}}
    output = output[1:]
    for line in output:
        try:
            opt, val, desc = re.match(
                r'\s+([^=]+)=(off|on): (.+)', line
            ).groups()
        except AttributeError:
            continue
        ret[pkg][opt] = val

    if not ret[pkg]:
        return {}
    return ret

def config(name, reset=False, **kwargs):
    '''
    Modify configuration options for a given port. Multiple options can be
    specified. To see the available options for a port, use
    :mod:`ports.showconfig <salt.modules.freebsdports.showconfig>`.

    name
        The port name, in ``category/name`` format

    reset : False
        If ``True``, runs a ``make rmconfig`` for the port, clearing its
        configuration before setting the desired options

    CLI Examples:

    .. code-block:: bash

        salt '*' ports.config security/nmap IPV6=off
    '''
    portpath = _check_portname(name)

    if reset:
        rmconfig(name)

    configuration = showconfig(name, dict_return=True)

    if not configuration:
        raise CommandExecutionError(
            'Unable to get port configuration for {0!r}'.format(name)
        )

    # Get top-level key for later reference
    pkg = next(iter(configuration))
    conf_ptr = configuration[pkg]

    opts = dict(
        (str(x), _normalize(kwargs[x]))
        for x in kwargs
        if not x.startswith('_')
    )

    bad_opts = [x for x in opts if x not in conf_ptr]
    if bad_opts:
        raise SaltInvocationError(
            'The following opts are not valid for port {0}: {1}'
            .format(name, ', '.join(bad_opts))
        )

    bad_vals = [
        '{0}={1}'.format(x, y) for x, y in six.iteritems(opts)
        if y not in ('on', 'off')
    ]
    if bad_vals:
        raise SaltInvocationError(
            'The following key/value pairs are invalid: {0}'
            .format(', '.join(bad_vals))
        )

    conf_ptr.update(opts)
    _write_options(name, configuration)

    new_config = showconfig(name, dict_return=True)
    try:
        new_config = new_config[next(iter(new_config))]
    except (StopIteration, TypeError):
        return False

    return all(conf_ptr[x] == new_config.get(x) for x in conf_ptr)

def update(extract=False):
    '''
    Update the ports tree

    extract : False
        If ``True``, runs a ``portsnap extract`` after fetching, should be used
        for first-time installation of the ports tree.

    CLI Example:

    .. code-block:: bash

        salt '*' ports.update
    '''
    result = __salt__['cmd.run_all']('{0} fetch'.format(_portsnap()))
    if not result['retcode'] == 0:
        raise CommandExecutionError(
            'Unable to fetch ports snapshot: {0}'.format(result['stderr'])
        )

    ret = []
    try:
        patch_count = re.search(
            r'Fetching (\d+) patches', result['stdout']
        ).group(1)
    except AttributeError:
        patch_count = 0

    try:
        new_port_count = re.search(
            r'Fetching (\d+) new ports or files', result['stdout']
        ).group(1)
    except AttributeError:
        new_port_count = 0

    ret.append('Applied {0} new patches'.format(patch_count))
    ret.append('Fetched {0} new ports or files'.format(new_port_count))

    if extract:
        result = __salt__['cmd.run_all']('{0} extract'.format(_portsnap()))
        if not result['retcode'] == 0:
            raise CommandExecutionError(
                'Unable to extract ports snapshot {0}'.format(result['stderr'])
            )

    result = __salt__['cmd.run_all']('{0} update'.format(_portsnap()))
    if not result['retcode'] == 0:
        raise CommandExecutionError(
            'Unable to apply ports snapshot: {0}'.format(result['stderr'])
        )

    __context__.pop('ports.list_all', None)
    return '\n'.join(ret)

def list_all():
    '''
    Lists all ports available.

    CLI Example:

    .. code-block:: bash

        salt '*' ports.list_all

    .. warning::

        Takes a while to run, and returns a **LOT** of output
    '''
    if 'ports.list_all' not in __context__:
        __context__['ports.list_all'] = []
        for path, dirs, files in os.walk('/usr/ports'):
            stripped = path[len('/usr/ports'):]
            if stripped.count('/') != 2 or stripped.endswith('/CVS'):
                continue
            __context__['ports.list_all'].append(stripped[1:])
    return __context__['ports.list_all']

def search(name):
    '''
    Search for matches in the ports tree. Globs are supported, and the category
    is optional

    CLI Examples:

    .. code-block:: bash

        salt '*' ports.search 'security/*'
        salt '*' ports.search 'security/n*'
        salt '*' ports.search nmap

    .. warning::

        Takes a while to run
    '''
    name = str(name)
    all_ports = list_all()
    if '/' in name:
        if name.count('/') > 1:
            raise SaltInvocationError(
                'Invalid search string {0!r}. Port names cannot have more '
                'than one slash'
            )
        else:
            return fnmatch.filter(all_ports, name)
    else:
        ret = []
        for port in all_ports:
            if fnmatch.fnmatch(port.rsplit('/')[-1], name):
                ret.append(port)
        return ret
<|endoftext|>


<commit_before>import os
import logging
import inspect

import funcy
import vivisect
import intervaltree

def getVwSampleMd5(vw):
    return vw.filemeta.values()[0]["md5sum"]

def getWorkspace(fp, reanalyze=False, verbose=False):
    '''
    For a file path return a workspace, it will create one if the extension
    is not .viv, otherwise it will load the existing one. Reanalyze will cause
    it to create and save a new one.
    '''
    vw = vivisect.VivWorkspace()
    vw.verbose = verbose
    vw.config.viv.parsers.pe.nx = True
    if fp.endswith('.viv'):
        vw.loadWorkspace(fp)
        if reanalyze:
            vw.analyze()
            vw.saveWorkspace()
    else:
        if os.path.exists(fp + ".viv"):
            vw.loadWorkspace(fp + ".viv")
            if reanalyze:
                vw.analyze()
                vw.saveWorkspace()
        else:
            vw.loadFromFile(fp)
            vw.analyze()
            vw.saveWorkspace()

    return vw

class LoggingObject(object):
    def __init__(self):
        self._logger = logging.getLogger("{:s}.{:s}".format(
            self.__module__, self.__class__.__name__))

    def _getCallerFunction(self):
        FUNCTION_NAME_INDEX = 3
        return inspect.stack()[3][FUNCTION_NAME_INDEX]

    def _formatFormatString(self, args):
        return [self._getCallerFunction() + ": " + args[0]] + [a for a in args[1:]]

    def d(self, *args, **kwargs):
        if self._logger.isEnabledFor(logging.DEBUG):
            self._logger.debug(*self._formatFormatString(args), **kwargs)

    def i(self, *args, **kwargs):
        if self._logger.isEnabledFor(logging.INFO):
            self._logger.info(*self._formatFormatString(args), **kwargs)

    def w(self, *args, **kwargs):
        if self._logger.isEnabledFor(logging.WARN):
            self._logger.warn(*self._formatFormatString(args), **kwargs)

    def e(self, *args, **kwargs):
        if self._logger.isEnabledFor(logging.ERROR):
            self._logger.error(*self._formatFormatString(args), **kwargs)

class Function(LoggingObject):
    def __init__(self, vw, va):
        super(Function, self).__init__()
        self.vw = vw
        self.va = va

    @funcy.cached_property
    def basic_blocks(self):
        bb = map(lambda b: BasicBlock(self.vw, *b), self.vw.getFunctionBlocks(self.va))
        return sorted(bb, key=lambda b: b.va)

    @funcy.cached_property
    def id(self):
        return self.vw.filemeta.values()[0]["md5sum"] + ":" + hex(self.va)

    def __repr__(self):
        return "Function(va: {:s})".format(hex(self.va))

class BasicBlock(LoggingObject):
    def __init__(self, vw, va, size, fva):
        super(BasicBlock, self).__init__()
        self.vw = vw
        self.va = va
        self.size = size
        self.fva = fva

    @funcy.cached_property
    def instructions(self):
        """
        from envi/__init__.py:class Opcode
        391         opcode   - An architecture specific numerical value for the opcode
        392         mnem     - A humon readable mnemonic for the opcode
        393         prefixes - a bitmask of architecture specific instruction prefixes
        394         size     - The size of the opcode in bytes
        395         operands - A list of Operand objects for this opcode
        396         iflags   - A list of Envi (architecture independant) instruction flags (see IF_FOO)
        397         va       - The virtual address the instruction lives at (used for PC relative im mediates etc...)
        """
        ret = []
        va = self.va
        while va < self.va + self.size:
            o = self.vw.parseOpcode(va)
            ret.append(o)
            va += len(o)
        return ret

    def __repr__(self):
        return "BasicBlock(va: {:s}, size: {:s}, fva: {:s})".format(
                hex(self.va), hex(self.size), hex(self.fva))

def one(s):
    for i in s:
        return i

class InstructionFunctionIndex(LoggingObject):
    """ Index from VA to containing function VA """
    def __init__(self, vw):
        super(InstructionFunctionIndex, self).__init__()
        self.vw = vw
        self._index = intervaltree.IntervalTree()
        self._do_index()

    def _do_index(self):
        for funcva in self.vw.getFunctions():
            f = Function(self.vw, funcva)
            for bb in f.basic_blocks:
                self._index[bb.va:bb.va + bb.size] = funcva

    def __getitem__(self, key):
        return one(self._index[key]).data

<commit_msg>function: add get/set name<commit_after>import os
import logging
import inspect

import funcy
import vivisect
import intervaltree

def getVwSampleMd5(vw):
    return vw.filemeta.values()[0]["md5sum"]

def getWorkspace(fp, reanalyze=False, verbose=False):
    '''
    For a file path return a workspace, it will create one if the extension
    is not .viv, otherwise it will load the existing one. Reanalyze will cause
    it to create and save a new one.
    '''
    vw = vivisect.VivWorkspace()
    vw.verbose = verbose
    vw.config.viv.parsers.pe.nx = True
    if fp.endswith('.viv'):
        vw.loadWorkspace(fp)
        if reanalyze:
            vw.analyze()
            vw.saveWorkspace()
    else:
        if os.path.exists(fp + ".viv"):
            vw.loadWorkspace(fp + ".viv")
            if reanalyze:
                vw.analyze()
                vw.saveWorkspace()
        else:
            vw.loadFromFile(fp)
            vw.analyze()
            vw.saveWorkspace()

    return vw

class LoggingObject(object):
    def __init__(self):
        self._logger = logging.getLogger("{:s}.{:s}".format(
            self.__module__, self.__class__.__name__))

    def _getCallerFunction(self):
        FUNCTION_NAME_INDEX = 3
        return inspect.stack()[3][FUNCTION_NAME_INDEX]

    def _formatFormatString(self, args):
        return [self._getCallerFunction() + ": " + args[0]] + [a for a in args[1:]]

    def d(self, *args, **kwargs):
        if self._logger.isEnabledFor(logging.DEBUG):
            self._logger.debug(*self._formatFormatString(args), **kwargs)

    def i(self, *args, **kwargs):
        if self._logger.isEnabledFor(logging.INFO):
            self._logger.info(*self._formatFormatString(args), **kwargs)

    def w(self, *args, **kwargs):
        if self._logger.isEnabledFor(logging.WARN):
            self._logger.warn(*self._formatFormatString(args), **kwargs)

    def e(self, *args, **kwargs):
        if self._logger.isEnabledFor(logging.ERROR):
            self._logger.error(*self._formatFormatString(args), **kwargs)

def set_function_name(vw, va, new_name):
    # vivgui seems to override function_name with symbol names, but this is correct
    ret_type, ret_name, call_conv, func_name, args = vw.getFunctionApi(va)
    vw.setFunctionApi(va, (ret_type, ret_name, call_conv, new_name, args))

def get_function_name(vw, va):
    ret_type, ret_name, call_conv, func_name, args = vw.getFunctionApi(va)
    return func_name

class Function(LoggingObject):
    def __init__(self, vw, va):
        super(Function, self).__init__()
        self.vw = vw
        self.va = va

    @funcy.cached_property
    def basic_blocks(self):
        bb = map(lambda b: BasicBlock(self.vw, *b), self.vw.getFunctionBlocks(self.va))
        return sorted(bb, key=lambda b: b.va)

    @funcy.cached_property
    def id(self):
        return self.vw.filemeta.values()[0]["md5sum"] + ":" + hex(self.va)

    def __repr__(self):
        return "Function(va: {:s})".format(hex(self.va))

    @property
    def name(self):
        return get_function_name(self.vw, self.va)

    @name.setter
    def name(self, new_name):
        return set_function_name(self.vw, self.va, new_name)

class BasicBlock(LoggingObject):
    def __init__(self, vw, va, size, fva):
        super(BasicBlock, self).__init__()
        self.vw = vw
        self.va = va
        self.size = size
        self.fva = fva

    @funcy.cached_property
    def instructions(self):
        """
        from envi/__init__.py:class Opcode
        391         opcode   - An architecture specific numerical value for the opcode
        392         mnem     - A humon readable mnemonic for the opcode
        393         prefixes - a bitmask of architecture specific instruction prefixes
        394         size     - The size of the opcode in bytes
        395         operands - A list of Operand objects for this opcode
        396         iflags   - A list of Envi (architecture independant) instruction flags (see IF_FOO)
        397         va       - The virtual address the instruction lives at (used for PC relative im mediates etc...)
        """
        ret = []
        va = self.va
        while va < self.va + self.size:
            o = self.vw.parseOpcode(va)
            ret.append(o)
            va += len(o)
        return ret

    def __repr__(self):
        return "BasicBlock(va: {:s}, size: {:s}, fva: {:s})".format(
                hex(self.va), hex(self.size), hex(self.fva))

def one(s):
    for i in s:
        return i

class InstructionFunctionIndex(LoggingObject):
    """ Index from VA to containing function VA """
    def __init__(self, vw):
        super(InstructionFunctionIndex, self).__init__()
        self.vw = vw
        self._index = intervaltree.IntervalTree()
        self._do_index()

    def _do_index(self):
        for funcva in self.vw.getFunctions():
            f = Function(self.vw, funcva)
            for bb in f.basic_blocks:
                self._index[bb.va:bb.va + bb.size] = funcva

    def __getitem__(self, key):
        return one(self._index[key]).data

<|endoftext|>


<commit_before>#=========================================================================
# Counter.py
#=========================================================================

from   pymtl import *
import pmlib

from   math import ceil, log

from   pmlib.TestVectorSimulator import TestVectorSimulator

# Counter Model

class Counter (Model):

  def __init__( s, max_count ):

    # Local Constants

    s.max_count = max_count
    s.nbits     = int( ceil( log( max_count + 1, 2 ) ) )

    # Interface

    s.increment = InPort  ( 1 )
    s.decrement = InPort  ( 1 )
    s.count     = OutPort ( s.nbits )
    s.zero      = OutPort ( 1 )

    # credit count register

    s.count_reg = pmlib.regs.RegRst( s.nbits, reset_value = max_count )
    connect( s.count_reg.out, s.count )

  @combinational
  def comb( s ):

    if   ( s.increment.value and ~s.decrement.value
         and ( s.count_reg.out.value < s.max_count ) ):
      s.count_reg.in_.value = s.count_reg.out.value + 1
    elif ( ~s.increment.value and s.decrement.value
         and ( s.count_reg.out.value > 0 ) ):
      s.count_reg.in_.value = s.count_reg.out.value - 1
    else:
      s.count_reg.in_.value = s.count_reg.out.value

    s.zero.value = ( s.count_reg.out.value == 0 )

# Unit tests for the counter

def test_counter( dump_vcd ):

  # Test vectors

  test_vectors = [
    # incr decr count zero
    [ 0,   0,   4,    0   ],
    [ 1,   1,   4,    0   ],
    [ 0,   1,   4,    0   ],
    [ 0,   1,   3,    0   ],
    [ 0,   1,   2,    0   ],
    [ 0,   1,   1,    0   ],
    [ 0,   1,   0,    1   ],
    [ 0,   1,   0,    1   ],
    [ 1,   0,   0,    1   ],
    [ 1,   0,   1,    0   ],
    [ 1,   0,   2,    0   ],
    [ 1,   0,   3,    0   ],
    [ 1,   0,   4,    0   ],
    [ 1,   0,   4,    0   ],
    [ 0,   1,   4,    0   ],
    [ 1,   1,   3,    0   ],
    [ 0,   0,   3,    0   ],
    [ 0,   1,   3,    0   ],
    [ 0,   1,   2,    0   ],
  ]

  # Instantiate and elaborate the model

  model = Counter(4)
  model.elaborate()

  # Define functions mapping the test vector to ports in model

  def tv_in( model, test_vector ):
    model.increment.value = test_vector[0]
    model.decrement.value = test_vector[1]

  def tv_out( model, test_vector ):
    assert model.count.value == test_vector[2]
    assert model.zero.value == test_vector[3]

  # Run the test

  sim = TestVectorSimulator( model, test_vectors, tv_in, tv_out )
  if dump_vcd:
    sim.dump_vcd( "Counter.vcd" )
  sim.run_test()
<commit_msg>[pex_dac] Counter model which passes verilation<commit_after>#=========================================================================
# Counter.py
#=========================================================================

from   pymtl import *
import pmlib

from   math import ceil, log

from   pmlib.TestVectorSimulator import TestVectorSimulator

# Counter Model

class Counter (Model):

  def __init__( s, max_count=4 ):

    # Local Constants

    s.max_count = max_count
    s.nbits     = int( ceil( log( max_count + 1, 2 ) ) )

    # Interface

    s.increment = InPort  ( 1 )
    s.decrement = InPort  ( 1 )
    s.count     = OutPort ( s.nbits )
    s.zero      = OutPort ( 1 )

    # credit count register

    s.count_reg = pmlib.regs.RegRst( s.nbits )
    connect( s.count_reg.out, s.count )

  @combinational
  def comb( s ):

    if   ( ( s.increment.value and (~s.decrement.value) )
         and ( s.count_reg.out.value < s.max_count ) ):
      s.count_reg.in_.value = s.count_reg.out.value + 1
    elif ( ( (~s.increment.value ) and s.decrement.value )
         and ( s.count_reg.out.value > 0 ) ):
      s.count_reg.in_.value = s.count_reg.out.value - 1
    else:
      s.count_reg.in_.value = s.count_reg.out.value

    s.zero.value = ( s.count_reg.out.value == 0 )

# Unit tests for the counter

def test_reg( dump_vcd ):

  # Test vectors

  test_vectors = [
    # incr decr count zero
    [ 0,   0,   0,    1   ],
    [ 1,   1,   0,    1   ],
    [ 1,   0,   0,    1   ],
    [ 1,   0,   1,    0   ],
    [ 1,   0,   2,    0   ],
    [ 1,   0,   3,    0   ],
    [ 1,   0,   4,    0   ],
    [ 1,   0,   4,    0   ],
    [ 0,   1,   4,    0   ],
    [ 0,   1,   3,    0   ],
    [ 0,   1,   2,    0   ],
    [ 0,   1,   1,    0   ],
    [ 0,   1,   0,    1   ],
    [ 0,   1,   0,    1   ],
    [ 1,   0,   0,    1   ],
    [ 1,   0,   1,    0   ],
    [ 1,   1,   2,    0   ],
    [ 1,   0,   2,    0   ],
  ]

  # Instantiate and elaborate the model

  model = Counter()
  model.elaborate()

  # Define functions mapping the test vector to ports in model

  def tv_in( model, test_vector ):
    model.increment.value = test_vector[0]
    model.decrement.value = test_vector[1]

  def tv_out( model, test_vector ):
    assert model.count.value == test_vector[2]
    assert model.zero.value == test_vector[3]

  # Run the test

  sim = TestVectorSimulator( model, test_vectors, tv_in, tv_out )
  if dump_vcd:
    sim.dump_vcd( "Counter.vcd" )
  sim.run_test()
<|endoftext|>


<commit_before>from datetime import datetime, date, timedelta
import unittest

from businesstime.holidays.aus import QueenslandPublicHolidays, BrisbanePublicHolidays

class QueenslandPublicHolidaysTest(unittest.TestCase):

    def test_2016_08(self):
        holidays_gen = QueenslandPublicHolidays()
        self.assertEqual(
            list(holidays_gen(date(2016, 8, 1), end=date(2016, 9, 1))),
            []
        )

class BrisbanePublicHolidaysTest(unittest.TestCase):

    def test_2016_08(self):
        holidays_gen = BrisbanePublicHolidays()
        self.assertEqual(
            list(holidays_gen(date(2016, 8, 1), end=date(2016, 9, 1))),
            [
                date(2016, 8, 10)
            ]
        )
<commit_msg>Addt'l aus test<commit_after>from datetime import datetime, date, timedelta
import unittest

from businesstime.holidays.aus import QueenslandPublicHolidays, BrisbanePublicHolidays

class QueenslandPublicHolidaysTest(unittest.TestCase):

    def test_2016_08(self):
        holidays_gen = QueenslandPublicHolidays()
        self.assertEqual(
            list(holidays_gen(date(2016, 8, 1), end=date(2016, 8, 31))),
            []
        )

class BrisbanePublicHolidaysTest(unittest.TestCase):

    def test_2016_08(self):
        holidays_gen = BrisbanePublicHolidays()
        self.assertEqual(
            list(holidays_gen(date(2016, 8, 1), end=date(2016, 8, 31))),
            [
                date(2016, 8, 10)
            ]
        )

    def test_out_of_range(self):
        holidays_gen = BrisbanePublicHolidays()
        with self.assertRaises(NotImplementedError):
            list(holidays_gen(date(2017, 1, 1), end=date(2017, 12, 31)))
<|endoftext|>


<commit_before>import numpy as np
import random as rand
import sys
import re
import string
from itertools import groupby

def get_crossword_batch(option, dimensions=64):
	if option == 'train':
		file_path = 'train.txt'
	if option == 'test':
		file_path = 'test.txt'
	if option == 'dev':
		file_path = 'dev.txt'

	with open(file_path, mode='r') as input_file:
		lines = list(input_file)
	rand.shuffle(lines)
	lines_batch = lines[0:64]

	tuple_list = []
	for line in lines_batch:
		line = line.replace(';',' ')
		words = line.split()
		cur_word = words[-1]
		tuple_list.append((cur_word, words[1:-1]))

	return tuple_list

def get_dictionary_batch(option, dimensions=64):
	if option == 'train':
		file_path = 'train.txt'
	if option == 'test':
		file_path = 'test.txt'
	if option == 'dev':
		file_path = 'dev.txt'

	with open(file_path, mode='r') as input_file:
		lines = list(input_file)
	rand.shuffle(lines)
	lines_batch = lines[0:64]

	tuple_list = []
	for line in lines_batch:
		while '  ' in line:
			line = line.replace('  ',' ')
		line = re.sub("[\(\[].*?[\)\]]", "", line)
		definitions = re.findall('\d+|\D+',line)
		definitions = [definition for definition in definitions if not definition.isdigit()]

		definitions_nopunct = []
		for definition in definitions:
			definition = ''.join(i for i in definition if i not in ('!',',','.','?',':',';'))
			definitions_nopunct.append(definition)

		tokenized_definitions = []
		for definition in definitions_nopunct:
			words = definition.split()

			# while 'v' in words: words.remove('v')
			# while 'n' in words: words.remove('n')
			# while 'adj' in words: words.remove('adj')
			# while 'adv' in words: words.remove('adv')
			# while 'prep' in words: words.remove('prep')
			# while 'symb' in words: words.remove('symb')
			# while 'abbr' in words: words.remove('abbr')

			tokenized_definitions.append(words)

		if len(tokenized_definitions) > 0 and len(tokenized_definitions[0]) > 0:
			cur_word = tokenized_definitions[0][0]

		tokenized_definitions[0] = tokenized_definitions[0][1:]

		for definition in tokenized_definitions:
			if len(definition) >= 2:
				tuple_list.append((cur_word,definition))

	return tuple_list

def main(argv):
	if argv[1] == 'dictionary':
		tuple_list = get_dictionary_batch(argv[2],64)
		print tuple_list
	if argv[1] == 'crossword':
		tuple_list = get_crossword_batch(argv[2],64)
		print tuple_list

if __name__ == "__main__":
	main(sys.argv)

<commit_msg>modified wrapper.py for double spaces<commit_after>import numpy as np
import random as rand
import sys
import re
import string
from itertools import groupby

def get_crossword_batch(option, dimensions=64):
	if option == 'train':
		file_path = 'train.txt'
	if option == 'test':
		file_path = 'test.txt'
	if option == 'dev':
		file_path = 'dev.txt'

	with open(file_path, mode='r') as input_file:
		lines = list(input_file)
	rand.shuffle(lines)
	lines_batch = lines[0:64]

	tuple_list = []
	for line in lines_batch:
		line = line.replace(';',' ')
		words = line.split()
		cur_word = words[-1]
		tuple_list.append((cur_word, words[:-1]))

	return tuple_list

def get_dictionary_batch(option, dimensions=64):
	if option == 'train':
		file_path = 'train.txt'
	if option == 'test':
		file_path = 'test.txt'
	if option == 'dev':
		file_path = 'dev.txt'

	with open(file_path, mode='r') as input_file:
		lines = list(input_file)
	rand.shuffle(lines)
	lines_batch = lines[0:64]

	tuple_list = []
	for line in lines_batch:
		while '  ' in line:
			line = line.replace('  ',' ')
		# line_words = line.split()
		# line = ' '.join(line_words)

		line = re.sub("[\(\[].*?[\)\]]", "", line)
		definitions = re.findall('\d+|\D+',line)
		definitions = [definition for definition in definitions if not definition.isdigit()]

		definitions_nopunct = []
		for definition in definitions:
			definition = ''.join(i for i in definition if i not in ('!',',','.','?',':',';'))
			definitions_nopunct.append(definition)

		tokenized_definitions = []
		for definition in definitions_nopunct:
			words = definition.split()
			tokenized_definitions.append(words)

		if len(tokenized_definitions) > 0 and len(tokenized_definitions[0]) > 0:
			cur_word = tokenized_definitions[0][0]

		if len(tokenized_definitions) > 1:
			tokenized_definitions[0] = tokenized_definitions[0][1:]
		else:
			#remove

		for definition in tokenized_definitions:
			if len(definition) >= 2:
				tuple_list.append((cur_word,definition))

	return tuple_list

def main(argv):
	if argv[1] == 'dictionary':
		tuple_list = get_dictionary_batch(argv[2],64)
		print tuple_list
	if argv[1] == 'crossword':
		tuple_list = get_crossword_batch(argv[2],64)
		print tuple_list

if __name__ == "__main__":
	main(sys.argv)

<|endoftext|>


<commit_before>#!/usr/bin/env python2

# TODO: Pick a license

"""
Converts django filters and tags
"""

import markdown, re
from django.template import Context, Template

class DjangoTemplateExtension(markdown.Extension):
    def __init__(self, configs={}):
        self.config = {
            'context': [Context({}), 'Rendering context instance'],
        }
        for key, value in configs:
            self.setConfig(key, value)

    def extendMarkdown(self, md, md_globals):
        """ Add DjangoTemplatePreprocessor to the Markdown instance. """

        ext = DjangoTemplatePreprocessor(md)
        ext.config = self.config
        md.preprocessors.add('django_template_block', ext, ">section_edit_block")

class DjangoTemplatePreprocessor(markdown.preprocessors.Preprocessor):

    def run(self, lines):
        text = "\n".join(lines)
        t = Template("{% load filters aatags %}" + text)
        c = self.config.get("context")[0]
        return t.render(c).split("\n")

def makeExtension(configs=None):
    return DjangoTemplateExtension()

if __name__ == "__main__":
    import doctest
    doctest.testmod()
<commit_msg>Removed mdx_django_template<commit_after><|endoftext|>


<commit_before>from django.http import FileResponse
from django.core.files.base import ContentFile

from rest_framework import generics
from rest_framework import permissions as drf_permissions
from rest_framework.exceptions import NotFound, PermissionDenied

from framework.auth.oauth_scopes import CoreScopes

from osf.models import (
    Guid,
    BaseFileNode,
    FileVersion,
    QuickFilesNode,
)

from api.base.exceptions import Gone, InvalidFilterValue
from api.base.permissions import PermissionWithGetter
from api.base.throttling import CreateGuidThrottle, NonCookieAuthThrottle, UserRateThrottle
from api.base import utils
from api.base.views import JSONAPIBaseView
from api.base import permissions as base_permissions
from api.nodes.permissions import ContributorOrPublic
from api.nodes.permissions import ReadOnlyIfRegistration
from api.files.permissions import IsPreprintFile
from api.files.permissions import CheckedOutOrAdmin
from api.files.serializers import FileSerializer
from api.files.serializers import FileDetailSerializer, QuickFilesDetailSerializer
from api.files.serializers import FileVersionSerializer
from api.files.serializers import FileMetadataRecordSerializer

class FileMixin(object):
    """Mixin with convenience methods for retrieving the current file based on the
    current URL. By default, fetches the file based on the file_id kwarg.
    """

    serializer_class = FileSerializer
    file_lookup_url_kwarg = 'file_id'

    def get_file(self, check_permissions=True):
        try:
            obj = utils.get_object_or_error(BaseFileNode, self.kwargs[self.file_lookup_url_kwarg], self.request, display_name='file')
        except NotFound:
            obj = utils.get_object_or_error(Guid, self.kwargs[self.file_lookup_url_kwarg], self.request).referent
            if obj.is_deleted:
                raise Gone(detail='The requested file is no longer available.')
            if not isinstance(obj, BaseFileNode):
                raise NotFound

        if getattr(obj.target, 'deleted', None):
            raise Gone(detail='The requested file is no longer available')

        if getattr(obj.target, 'is_quickfiles', False) and getattr(obj.target, 'creator'):
            if obj.target.creator.is_disabled:
                raise Gone(detail='This user has been deactivated and their quickfiles are no longer available.')

        if check_permissions:
            # May raise a permission denied
            self.check_object_permissions(self.request, obj)
        return obj

class FileDetail(JSONAPIBaseView, generics.RetrieveUpdateAPIView, FileMixin):
    """The documentation for this endpoint can be found [here](https://developer.osf.io/#operation/files_detail).
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        IsPreprintFile,
        CheckedOutOrAdmin,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
        PermissionWithGetter(ReadOnlyIfRegistration, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = FileDetailSerializer
    throttle_classes = (CreateGuidThrottle, NonCookieAuthThrottle, UserRateThrottle, )
    view_category = 'files'
    view_name = 'file-detail'

    def get_serializer_class(self):
        try:
            target = self.get_target()
        except (NotFound, Gone, PermissionDenied):
            return FileDetailSerializer
        else:
            if isinstance(target, QuickFilesNode):
                return QuickFilesDetailSerializer
            return FileDetailSerializer

    def get_target(self):
        return self.get_file().target

    # overrides RetrieveAPIView
    def get_object(self):
        user = utils.get_user_auth(self.request).user
        file = self.get_file()

        if self.request.GET.get('create_guid', False):
            # allows quickfiles to be given guids when another user wants a permanent link to it
            if (self.get_target().has_permission(user, 'admin') and utils.has_admin_scope(self.request)) or getattr(file.target, 'is_quickfiles', False):
                file.get_guid(create=True)
        return file

class FileVersionsList(JSONAPIBaseView, generics.ListAPIView, FileMixin):
    """The documentation for this endpoint can be found [here](https://developer.osf.io/#operation/files_versions).
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = FileVersionSerializer
    view_category = 'files'
    view_name = 'file-versions'

    ordering = ('-modified',)

    def get_queryset(self):
        self.file = self.get_file()
        return self.file.versions.all()

    def get_serializer_context(self):
        context = JSONAPIBaseView.get_serializer_context(self)
        context['file'] = self.file
        return context

def node_from_version(request, view, obj):
    return view.get_file(check_permissions=False).target

class FileVersionDetail(JSONAPIBaseView, generics.RetrieveAPIView, FileMixin):
    """The documentation for this endpoint can be found [here](https://developer.osf.io/#operation/files_version_detail).
    """
    version_lookup_url_kwarg = 'version_id'
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, node_from_version),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = FileVersionSerializer
    view_category = 'files'
    view_name = 'version-detail'

    # overrides RetrieveAPIView
    def get_object(self):
        self.file = self.get_file()
        maybe_version = self.file.get_version(self.kwargs[self.version_lookup_url_kwarg])

        # May raise a permission denied
        # Kinda hacky but versions have no reference to node or file
        self.check_object_permissions(self.request, self.file)
        return utils.get_object_or_error(FileVersion, getattr(maybe_version, '_id', ''), self.request)

    def get_serializer_context(self):
        context = JSONAPIBaseView.get_serializer_context(self)
        context['file'] = self.file
        return context

class FileMetadataRecordsList(JSONAPIBaseView, generics.ListAPIView, FileMixin):

    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NULL]

    serializer_class = FileMetadataRecordSerializer
    view_category = 'files'
    view_name = 'metadata-records'

    ordering = ('-created',)

    def get_queryset(self):
        return self.get_file().records.all()

class FileMetadataRecordDetail(JSONAPIBaseView, generics.RetrieveUpdateAPIView, FileMixin):

    record_lookup_url_kwarg = 'record_id'
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
        PermissionWithGetter(ReadOnlyIfRegistration, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = FileMetadataRecordSerializer
    view_category = 'files'
    view_name = 'metadata-record-detail'

    def get_object(self):
        return utils.get_object_or_error(
            self.get_file().records.filter(_id=self.kwargs[self.record_lookup_url_kwarg]),
            request=self.request,
        )

class FileMetadataRecordDownload(JSONAPIBaseView, generics.RetrieveAPIView, FileMixin):

    record_lookup_url_kwarg = 'record_id'
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NULL]

    view_category = 'files'
    view_name = 'metadata-record-download'

    def get_serializer_class(self):
        return None

    def get_object(self):
        return utils.get_object_or_error(
            self.get_file().records.filter(_id=self.kwargs[self.record_lookup_url_kwarg]).select_related('schema', 'file'),
            request=self.request,
        )

    def get(self, request, **kwargs):
        file_type = self.request.query_params.get('export', 'json')
        record = self.get_object()
        try:
            response = FileResponse(ContentFile(record.serialize(format=file_type)))
        except ValueError as e:
            raise InvalidFilterValue(detail=str(e))
        file_name = 'file_metadata_{}.{}'.format(record.schema._id, file_type)
        response['Content-Disposition'] = 'attachment; filename="{}"'.format(file_name)
        response['Content-Type'] = 'application/{}'.format(file_type)
        return response
<commit_msg>Add filename to output file for file metadata<commit_after>from django.http import FileResponse
from django.core.files.base import ContentFile

from rest_framework import generics
from rest_framework import permissions as drf_permissions
from rest_framework.exceptions import NotFound, PermissionDenied

from framework.auth.oauth_scopes import CoreScopes

from osf.models import (
    Guid,
    BaseFileNode,
    FileVersion,
    QuickFilesNode,
)

from api.base.exceptions import Gone, InvalidFilterValue
from api.base.permissions import PermissionWithGetter
from api.base.throttling import CreateGuidThrottle, NonCookieAuthThrottle, UserRateThrottle
from api.base import utils
from api.base.views import JSONAPIBaseView
from api.base import permissions as base_permissions
from api.nodes.permissions import ContributorOrPublic
from api.nodes.permissions import ReadOnlyIfRegistration
from api.files.permissions import IsPreprintFile
from api.files.permissions import CheckedOutOrAdmin
from api.files.serializers import FileSerializer
from api.files.serializers import FileDetailSerializer, QuickFilesDetailSerializer
from api.files.serializers import FileVersionSerializer
from api.files.serializers import FileMetadataRecordSerializer

class FileMixin(object):
    """Mixin with convenience methods for retrieving the current file based on the
    current URL. By default, fetches the file based on the file_id kwarg.
    """

    serializer_class = FileSerializer
    file_lookup_url_kwarg = 'file_id'

    def get_file(self, check_permissions=True):
        try:
            obj = utils.get_object_or_error(BaseFileNode, self.kwargs[self.file_lookup_url_kwarg], self.request, display_name='file')
        except NotFound:
            obj = utils.get_object_or_error(Guid, self.kwargs[self.file_lookup_url_kwarg], self.request).referent
            if obj.is_deleted:
                raise Gone(detail='The requested file is no longer available.')
            if not isinstance(obj, BaseFileNode):
                raise NotFound

        if getattr(obj.target, 'deleted', None):
            raise Gone(detail='The requested file is no longer available')

        if getattr(obj.target, 'is_quickfiles', False) and getattr(obj.target, 'creator'):
            if obj.target.creator.is_disabled:
                raise Gone(detail='This user has been deactivated and their quickfiles are no longer available.')

        if check_permissions:
            # May raise a permission denied
            self.check_object_permissions(self.request, obj)
        return obj

class FileDetail(JSONAPIBaseView, generics.RetrieveUpdateAPIView, FileMixin):
    """The documentation for this endpoint can be found [here](https://developer.osf.io/#operation/files_detail).
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        IsPreprintFile,
        CheckedOutOrAdmin,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
        PermissionWithGetter(ReadOnlyIfRegistration, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = FileDetailSerializer
    throttle_classes = (CreateGuidThrottle, NonCookieAuthThrottle, UserRateThrottle, )
    view_category = 'files'
    view_name = 'file-detail'

    def get_serializer_class(self):
        try:
            target = self.get_target()
        except (NotFound, Gone, PermissionDenied):
            return FileDetailSerializer
        else:
            if isinstance(target, QuickFilesNode):
                return QuickFilesDetailSerializer
            return FileDetailSerializer

    def get_target(self):
        return self.get_file().target

    # overrides RetrieveAPIView
    def get_object(self):
        user = utils.get_user_auth(self.request).user
        file = self.get_file()

        if self.request.GET.get('create_guid', False):
            # allows quickfiles to be given guids when another user wants a permanent link to it
            if (self.get_target().has_permission(user, 'admin') and utils.has_admin_scope(self.request)) or getattr(file.target, 'is_quickfiles', False):
                file.get_guid(create=True)
        return file

class FileVersionsList(JSONAPIBaseView, generics.ListAPIView, FileMixin):
    """The documentation for this endpoint can be found [here](https://developer.osf.io/#operation/files_versions).
    """
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = FileVersionSerializer
    view_category = 'files'
    view_name = 'file-versions'

    ordering = ('-modified',)

    def get_queryset(self):
        self.file = self.get_file()
        return self.file.versions.all()

    def get_serializer_context(self):
        context = JSONAPIBaseView.get_serializer_context(self)
        context['file'] = self.file
        return context

def node_from_version(request, view, obj):
    return view.get_file(check_permissions=False).target

class FileVersionDetail(JSONAPIBaseView, generics.RetrieveAPIView, FileMixin):
    """The documentation for this endpoint can be found [here](https://developer.osf.io/#operation/files_version_detail).
    """
    version_lookup_url_kwarg = 'version_id'
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, node_from_version),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = FileVersionSerializer
    view_category = 'files'
    view_name = 'version-detail'

    # overrides RetrieveAPIView
    def get_object(self):
        self.file = self.get_file()
        maybe_version = self.file.get_version(self.kwargs[self.version_lookup_url_kwarg])

        # May raise a permission denied
        # Kinda hacky but versions have no reference to node or file
        self.check_object_permissions(self.request, self.file)
        return utils.get_object_or_error(FileVersion, getattr(maybe_version, '_id', ''), self.request)

    def get_serializer_context(self):
        context = JSONAPIBaseView.get_serializer_context(self)
        context['file'] = self.file
        return context

class FileMetadataRecordsList(JSONAPIBaseView, generics.ListAPIView, FileMixin):

    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NULL]

    serializer_class = FileMetadataRecordSerializer
    view_category = 'files'
    view_name = 'metadata-records'

    ordering = ('-created',)

    def get_queryset(self):
        return self.get_file().records.all()

class FileMetadataRecordDetail(JSONAPIBaseView, generics.RetrieveUpdateAPIView, FileMixin):

    record_lookup_url_kwarg = 'record_id'
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
        PermissionWithGetter(ReadOnlyIfRegistration, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NODE_FILE_WRITE]

    serializer_class = FileMetadataRecordSerializer
    view_category = 'files'
    view_name = 'metadata-record-detail'

    def get_object(self):
        return utils.get_object_or_error(
            self.get_file().records.filter(_id=self.kwargs[self.record_lookup_url_kwarg]),
            request=self.request,
        )

class FileMetadataRecordDownload(JSONAPIBaseView, generics.RetrieveAPIView, FileMixin):

    record_lookup_url_kwarg = 'record_id'
    permission_classes = (
        drf_permissions.IsAuthenticatedOrReadOnly,
        base_permissions.TokenHasScope,
        PermissionWithGetter(ContributorOrPublic, 'target'),
    )

    required_read_scopes = [CoreScopes.NODE_FILE_READ]
    required_write_scopes = [CoreScopes.NULL]

    view_category = 'files'
    view_name = 'metadata-record-download'

    def get_serializer_class(self):
        return None

    def get_object(self):
        return utils.get_object_or_error(
            self.get_file().records.filter(_id=self.kwargs[self.record_lookup_url_kwarg]).select_related('schema', 'file'),
            request=self.request,
        )

    def get(self, request, **kwargs):
        file_type = self.request.query_params.get('export', 'json')
        record = self.get_object()
        try:
            response = FileResponse(ContentFile(record.serialize(format=file_type)))
        except ValueError as e:
            raise InvalidFilterValue(detail=str(e))
        file_name = 'file_metadata_{}_{}.{}'.format(record.schema._id, record.file.name, file_type)
        response['Content-Disposition'] = 'attachment; filename="{}"'.format(file_name)
        response['Content-Type'] = 'application/{}'.format(file_type)
        return response
<|endoftext|>


<commit_before>import logging
import re
import tempfile

from django.conf import settings

from django_statsd.clients import statsd

from .utils import check_output, subprocess, VideoBase

log = logging.getLogger('z.video')

format_re = re.compile('TOTEM_INFO_VIDEO_CODEC=([\w]+)')
duration_re = re.compile('TOTEM_INFO_DURATION=(\d+)')

class Video(VideoBase):
    name = settings.TOTEM_BINARIES

    def _call_indexer(self):
        with statsd.timer('video.totem.meta'):
            args = [self.name['indexer'],
                    self.filename]
            log.info('totem called with: %s' % ' '.join(args))
            try:
                res = check_output(args, stderr=subprocess.STDOUT)
            except subprocess.CalledProcessError, e:
                log.error('totem failed with: %s' % e.output)
                raise
        return res

    def _call_thumbnailer(self, timepoint, destination, size):
        with statsd.timer('video.totem.screenshot'):
            args = [self.name['thumbnailer'],
                    '-t', timepoint,  # Start at this point.
                    '-s', size,  # This can only be one of the sizes.
                    self.filename,
                    destination]
            log.info('totem called with: %s' % ' '.join(args))
            try:
                res = check_output(args, stderr=subprocess.STDOUT)
            except subprocess.CalledProcessError, e:
                log.error('totem failed with: %s' % e.output)
                raise
        return res

    def get_meta(self):
        result = self._call_indexer()
        data = {}
        formats = format_re.search(result)
        if formats:
            data['formats'] = formats.group(1)

        duration = duration_re.search(result)
        if duration:
            data['duration'] = duration.group(1)

        self.meta = data

    def get_screenshot(self, size):
        assert self.is_valid()
        assert self.meta.get('duration')
        halfway = int(self.meta['duration'] / 2)
        dest = tempfile.mkstemp(suffix='.png')[1]
        self._call_thumbnailer(halfway, dest, max(size))
        return dest

    def is_valid(self):
        assert self.meta is not None
        self.errors = []
        if 'VP8' not in self.meta.get('formats', ''):
            self.errors.append('Videos must be WEBM.')

        #TODO(andym): More checks on duration, file size, bit rate?
        return not self.errors

    @classmethod
    def library_available(cls):
        try:
            # We'll assume if the thumbnailer is there so is the indexer.
            check_output([cls.name['thumbnailer'], '-help'],
                         stderr=subprocess.STDOUT)
            return True
        except (OSError, subprocess.CalledProcessError):
            pass
<commit_msg>divide properly<commit_after>import logging
import re
import tempfile

from django.conf import settings

from django_statsd.clients import statsd

from .utils import check_output, subprocess, VideoBase

log = logging.getLogger('z.video')

format_re = re.compile('TOTEM_INFO_VIDEO_CODEC=([\w]+)')
duration_re = re.compile('TOTEM_INFO_DURATION=(\d+)')

class Video(VideoBase):
    name = settings.TOTEM_BINARIES

    def _call_indexer(self):
        with statsd.timer('video.totem.meta'):
            args = [self.name['indexer'],
                    self.filename]
            log.info('totem called with: %s' % ' '.join(args))
            try:
                res = check_output(args, stderr=subprocess.STDOUT)
            except subprocess.CalledProcessError, e:
                log.error('totem failed with: %s' % e.output)
                raise
        return res

    def _call_thumbnailer(self, timepoint, destination, size):
        with statsd.timer('video.totem.screenshot'):
            args = [self.name['thumbnailer'],
                    '-t', timepoint,  # Start at this point.
                    '-s', size,  # This can only be one of the sizes.
                    self.filename,
                    destination]
            log.info('totem called with: %s' % ' '.join(args))
            try:
                res = check_output(args, stderr=subprocess.STDOUT)
            except subprocess.CalledProcessError, e:
                log.error('totem failed with: %s' % e.output)
                raise
        return res

    def get_meta(self):
        result = self._call_indexer()
        data = {}
        formats = format_re.search(result)
        if formats:
            data['formats'] = formats.group(1)

        duration = duration_re.search(result)
        if duration:
            data['duration'] = duration.group(1)

        self.meta = data

    def get_screenshot(self, size):
        assert self.is_valid()
        assert self.meta.get('duration')
        halfway = int(self.meta['duration']) / 2
        dest = tempfile.mkstemp(suffix='.png')[1]
        self._call_thumbnailer(halfway, dest, max(size))
        return dest

    def is_valid(self):
        assert self.meta is not None
        self.errors = []
        if 'VP8' not in self.meta.get('formats', ''):
            self.errors.append('Videos must be WEBM.')

        #TODO(andym): More checks on duration, file size, bit rate?
        return not self.errors

    @classmethod
    def library_available(cls):
        try:
            # We'll assume if the thumbnailer is there so is the indexer.
            check_output([cls.name['thumbnailer'], '-help'],
                         stderr=subprocess.STDOUT)
            return True
        except (OSError, subprocess.CalledProcessError):
            pass
<|endoftext|>


<commit_before><commit_msg>Remove unneessary import<commit_after><|endoftext|>