<commit_before>""" Misc low-level representation of HDF5 objects. """

import struct
from collections import OrderedDict

from .core import _padded_size, _structure_size
from .core import _unpack_struct_from, _unpack_struct_from_file
from .core import InvalidHDF5File

class SuperBlock(object):
    """
    HDF5 Superblock.
    """

    def __init__(self, fh, offset):
        """ initalize. """

        fh.seek(offset)
        version_hint = struct.unpack_from('<B', fh.read(9), 8)[0]
        fh.seek(offset)
        if version_hint == 0:
            contents = _unpack_struct_from_file(SUPERBLOCK_V0, fh)
            self._end_of_sblock = offset + SUPERBLOCK_V0_SIZE
        elif version_hint == 2 or version_hint == 3:
            contents = _unpack_struct_from_file(SUPERBLOCK_V2_V3, fh)
            self._end_of_sblock = offset + SUPERBLOCK_V2_V3_SIZE
        else:
            raise NotImplementedError(
                "unsupported superblock version: %i" % (version_hint))

        # verify contents
        if contents['format_signature'] != FORMAT_SIGNATURE:
            raise InvalidHDF5File('Incorrect file signature')
        if contents['offset_size'] != 8 or contents['length_size'] != 8:
            raise NotImplementedError('File uses none 64-bit addressing')
        self.version = contents['superblock_version']
        self._contents = contents
        self._root_symbol_table = None
        self._fh = fh

    @property
    def offset_to_dataobjects(self):
        """ The offset to the data objects collection for the superblock. """
        if self.version == 0:
            sym_table = SymbolTable(self._fh, self._end_of_sblock, root=True)
            self._root_symbol_table = sym_table
            return sym_table.group_offset
        elif self.version == 2 or version_hint == 3:
            return self._contents['root_group_address']
        else:
            raise NotImplementedError

class Heap(object):
    """
    HDF5 local heap.
    """

    def __init__(self, fh, offset):
        """ initalize. """

        fh.seek(offset)
        local_heap = _unpack_struct_from_file(LOCAL_HEAP, fh)
        assert local_heap['signature'] == b'HEAP'
        assert local_heap['version'] == 0
        fh.seek(local_heap['address_of_data_segment'])
        heap_data = fh.read(local_heap['data_segment_size'])
        local_heap['heap_data'] = heap_data
        self._contents = local_heap
        self.data = heap_data

    def get_object_name(self, offset):
        """ Return the name of the object indicated by the given offset. """
        end = self.data.index(b'\x00', offset)
        return self.data[offset:end]

class SymbolTable(object):
    """
    HDF5 Symbol Table.
    """

    def __init__(self, fh, offset, root=False):
        """ initialize, root=True for the root group, False otherwise. """

        fh.seek(offset)
        if root:
            # The root symbol table has no Symbol table node header
            # and contains only a single entry
            node = OrderedDict([('symbols', 1)])
        else:
            node = _unpack_struct_from_file(SYMBOL_TABLE_NODE, fh)
            assert node['signature'] == b'SNOD'
        entries = [_unpack_struct_from_file(SYMBOL_TABLE_ENTRY, fh) for i in
                   range(node['symbols'])]
        if root:
            self.group_offset = entries[0]['object_header_address']
        self.entries = entries
        self._contents = node

    def assign_name(self, heap):
        """ Assign link names to all entries in the symbol table. """
        for entry in self.entries:
            offset = entry['link_name_offset']
            link_name = heap.get_object_name(offset).decode('utf-8')
            entry['link_name'] = link_name
        return

    def get_links(self):
        """ Return a dictionary of links (dataset/group) and offsets. """
        return {e['link_name']: e['object_header_address'] for e in
                self.entries}

class GlobalHeap(object):
    """
    HDF5 Global Heap collection.
    """

    def __init__(self, fh, offset):

        fh.seek(offset)
        header = _unpack_struct_from_file(GLOBAL_HEAP_HEADER, fh)
        assert header['signature'] == b'GCOL'
        assert header['version'] == 1
        heap_data_size = header['collection_size'] - GLOBAL_HEAP_HEADER_SIZE
        heap_data = fh.read(heap_data_size)
        assert len(heap_data) == heap_data_size  # check for early end of file

        self.heap_data = heap_data
        self._header = header
        self._objects = None

    @property
    def objects(self):
        """ Dictionary of objects in the heap. """
        if self._objects is None:
            self._objects = OrderedDict()
            offset = 0
            while offset < len(self.heap_data):
                info = _unpack_struct_from(
                    GLOBAL_HEAP_OBJECT, self.heap_data, offset)
                if info['object_index'] == 0:
                    break
                offset += GLOBAL_HEAP_OBJECT_SIZE
                fmt = '<' + str(info['object_size']) + 's'
                obj_data = struct.unpack_from(fmt, self.heap_data, offset)[0]
                self._objects[info['object_index']] = obj_data
                offset += _padded_size(info['object_size'])
        return self._objects

FORMAT_SIGNATURE = b'\211HDF\r\n\032\n'

# Version 0 SUPERBLOCK
SUPERBLOCK_V0 = OrderedDict((
    ('format_signature', '8s'),

    ('superblock_version', 'B'),
    ('free_storage_version', 'B'),
    ('root_group_version', 'B'),
    ('reserved_0', 'B'),

    ('shared_header_version', 'B'),
    ('offset_size', 'B'),            # assume 8
    ('length_size', 'B'),            # assume 8
    ('reserved_1', 'B'),

    ('group_leaf_node_k', 'H'),
    ('group_internal_node_k', 'H'),

    ('file_consistency_flags', 'L'),

    ('base_address', 'Q'),                  # assume 8 byte addressing
    ('free_space_address', 'Q'),            # assume 8 byte addressing
    ('end_of_file_address', 'Q'),           # assume 8 byte addressing
    ('driver_information_address', 'Q'),    # assume 8 byte addressing

))
SUPERBLOCK_V0_SIZE = _structure_size(SUPERBLOCK_V0)

# Version 2 and 3 SUPERBLOCK
SUPERBLOCK_V2_V3 = OrderedDict((
    ('format_signature', '8s'),

    ('superblock_version', 'B'),
    ('offset_size', 'B'),
    ('length_size', 'B'),
    ('file_consistency_flags', 'B'),

    ('base_address', 'Q'),                  # assume 8 byte addressing
    ('superblock_extension_address', 'Q'),  # assume 8 byte addressing
    ('end_of_file_address', 'Q'),           # assume 8 byte addressing
    ('root_group_address', 'Q'),            # assume 8 byte addressing

    ('superblock_checksum', 'I'),

))
SUPERBLOCK_V2_V3_SIZE = _structure_size(SUPERBLOCK_V2_V3)

SYMBOL_TABLE_NODE = OrderedDict((
    ('signature', '4s'),
    ('version', 'B'),
    ('reserved_0', 'B'),
    ('symbols', 'H'),
))

SYMBOL_TABLE_ENTRY = OrderedDict((
    ('link_name_offset', 'Q'),     # 8 byte address
    ('object_header_address', 'Q'),
    ('cache_type', 'I'),
    ('reserved', 'I'),
    ('scratch', '16s'),
))

# III.D Disk Format: Level 1D - Local Heaps
LOCAL_HEAP = OrderedDict((
    ('signature', '4s'),
    ('version', 'B'),
    ('reserved', '3s'),
    ('data_segment_size', 'Q'),         # 8 byte size of lengths
    ('offset_to_free_list', 'Q'),       # 8 bytes size of lengths
    ('address_of_data_segment', 'Q'),   # 8 byte addressing
))

# III.E Disk Format: Level 1E - Global Heap
GLOBAL_HEAP_HEADER = OrderedDict((
    ('signature', '4s'),
    ('version', 'B'),
    ('reserved', '3s'),
    ('collection_size', 'Q'),
))
GLOBAL_HEAP_HEADER_SIZE = _structure_size(GLOBAL_HEAP_HEADER)

GLOBAL_HEAP_OBJECT = OrderedDict((
    ('object_index', 'H'),
    ('reference_count', 'H'),
    ('reserved', 'I'),
    ('object_size', 'Q')    # 8 byte addressing
))
GLOBAL_HEAP_OBJECT_SIZE = _structure_size(GLOBAL_HEAP_OBJECT)
<commit_msg>BUG: fix reading of superblock version 3<commit_after>""" Misc low-level representation of HDF5 objects. """

import struct
from collections import OrderedDict

from .core import _padded_size, _structure_size
from .core import _unpack_struct_from, _unpack_struct_from_file
from .core import InvalidHDF5File

class SuperBlock(object):
    """
    HDF5 Superblock.
    """

    def __init__(self, fh, offset):
        """ initalize. """

        fh.seek(offset)
        version_hint = struct.unpack_from('<B', fh.read(9), 8)[0]
        fh.seek(offset)
        if version_hint == 0:
            contents = _unpack_struct_from_file(SUPERBLOCK_V0, fh)
            self._end_of_sblock = offset + SUPERBLOCK_V0_SIZE
        elif version_hint == 2 or version_hint == 3:
            contents = _unpack_struct_from_file(SUPERBLOCK_V2_V3, fh)
            self._end_of_sblock = offset + SUPERBLOCK_V2_V3_SIZE
        else:
            raise NotImplementedError(
                "unsupported superblock version: %i" % (version_hint))

        # verify contents
        if contents['format_signature'] != FORMAT_SIGNATURE:
            raise InvalidHDF5File('Incorrect file signature')
        if contents['offset_size'] != 8 or contents['length_size'] != 8:
            raise NotImplementedError('File uses none 64-bit addressing')
        self.version = contents['superblock_version']
        self._contents = contents
        self._root_symbol_table = None
        self._fh = fh

    @property
    def offset_to_dataobjects(self):
        """ The offset to the data objects collection for the superblock. """
        if self.version == 0:
            sym_table = SymbolTable(self._fh, self._end_of_sblock, root=True)
            self._root_symbol_table = sym_table
            return sym_table.group_offset
        elif self.version == 2 or self.version == 3:
            return self._contents['root_group_address']
        else:
            raise NotImplementedError

class Heap(object):
    """
    HDF5 local heap.
    """

    def __init__(self, fh, offset):
        """ initalize. """

        fh.seek(offset)
        local_heap = _unpack_struct_from_file(LOCAL_HEAP, fh)
        assert local_heap['signature'] == b'HEAP'
        assert local_heap['version'] == 0
        fh.seek(local_heap['address_of_data_segment'])
        heap_data = fh.read(local_heap['data_segment_size'])
        local_heap['heap_data'] = heap_data
        self._contents = local_heap
        self.data = heap_data

    def get_object_name(self, offset):
        """ Return the name of the object indicated by the given offset. """
        end = self.data.index(b'\x00', offset)
        return self.data[offset:end]

class SymbolTable(object):
    """
    HDF5 Symbol Table.
    """

    def __init__(self, fh, offset, root=False):
        """ initialize, root=True for the root group, False otherwise. """

        fh.seek(offset)
        if root:
            # The root symbol table has no Symbol table node header
            # and contains only a single entry
            node = OrderedDict([('symbols', 1)])
        else:
            node = _unpack_struct_from_file(SYMBOL_TABLE_NODE, fh)
            assert node['signature'] == b'SNOD'
        entries = [_unpack_struct_from_file(SYMBOL_TABLE_ENTRY, fh) for i in
                   range(node['symbols'])]
        if root:
            self.group_offset = entries[0]['object_header_address']
        self.entries = entries
        self._contents = node

    def assign_name(self, heap):
        """ Assign link names to all entries in the symbol table. """
        for entry in self.entries:
            offset = entry['link_name_offset']
            link_name = heap.get_object_name(offset).decode('utf-8')
            entry['link_name'] = link_name
        return

    def get_links(self):
        """ Return a dictionary of links (dataset/group) and offsets. """
        return {e['link_name']: e['object_header_address'] for e in
                self.entries}

class GlobalHeap(object):
    """
    HDF5 Global Heap collection.
    """

    def __init__(self, fh, offset):

        fh.seek(offset)
        header = _unpack_struct_from_file(GLOBAL_HEAP_HEADER, fh)
        assert header['signature'] == b'GCOL'
        assert header['version'] == 1
        heap_data_size = header['collection_size'] - GLOBAL_HEAP_HEADER_SIZE
        heap_data = fh.read(heap_data_size)
        assert len(heap_data) == heap_data_size  # check for early end of file

        self.heap_data = heap_data
        self._header = header
        self._objects = None

    @property
    def objects(self):
        """ Dictionary of objects in the heap. """
        if self._objects is None:
            self._objects = OrderedDict()
            offset = 0
            while offset < len(self.heap_data):
                info = _unpack_struct_from(
                    GLOBAL_HEAP_OBJECT, self.heap_data, offset)
                if info['object_index'] == 0:
                    break
                offset += GLOBAL_HEAP_OBJECT_SIZE
                fmt = '<' + str(info['object_size']) + 's'
                obj_data = struct.unpack_from(fmt, self.heap_data, offset)[0]
                self._objects[info['object_index']] = obj_data
                offset += _padded_size(info['object_size'])
        return self._objects

FORMAT_SIGNATURE = b'\211HDF\r\n\032\n'

# Version 0 SUPERBLOCK
SUPERBLOCK_V0 = OrderedDict((
    ('format_signature', '8s'),

    ('superblock_version', 'B'),
    ('free_storage_version', 'B'),
    ('root_group_version', 'B'),
    ('reserved_0', 'B'),

    ('shared_header_version', 'B'),
    ('offset_size', 'B'),            # assume 8
    ('length_size', 'B'),            # assume 8
    ('reserved_1', 'B'),

    ('group_leaf_node_k', 'H'),
    ('group_internal_node_k', 'H'),

    ('file_consistency_flags', 'L'),

    ('base_address', 'Q'),                  # assume 8 byte addressing
    ('free_space_address', 'Q'),            # assume 8 byte addressing
    ('end_of_file_address', 'Q'),           # assume 8 byte addressing
    ('driver_information_address', 'Q'),    # assume 8 byte addressing

))
SUPERBLOCK_V0_SIZE = _structure_size(SUPERBLOCK_V0)

# Version 2 and 3 SUPERBLOCK
SUPERBLOCK_V2_V3 = OrderedDict((
    ('format_signature', '8s'),

    ('superblock_version', 'B'),
    ('offset_size', 'B'),
    ('length_size', 'B'),
    ('file_consistency_flags', 'B'),

    ('base_address', 'Q'),                  # assume 8 byte addressing
    ('superblock_extension_address', 'Q'),  # assume 8 byte addressing
    ('end_of_file_address', 'Q'),           # assume 8 byte addressing
    ('root_group_address', 'Q'),            # assume 8 byte addressing

    ('superblock_checksum', 'I'),

))
SUPERBLOCK_V2_V3_SIZE = _structure_size(SUPERBLOCK_V2_V3)

SYMBOL_TABLE_NODE = OrderedDict((
    ('signature', '4s'),
    ('version', 'B'),
    ('reserved_0', 'B'),
    ('symbols', 'H'),
))

SYMBOL_TABLE_ENTRY = OrderedDict((
    ('link_name_offset', 'Q'),     # 8 byte address
    ('object_header_address', 'Q'),
    ('cache_type', 'I'),
    ('reserved', 'I'),
    ('scratch', '16s'),
))

# III.D Disk Format: Level 1D - Local Heaps
LOCAL_HEAP = OrderedDict((
    ('signature', '4s'),
    ('version', 'B'),
    ('reserved', '3s'),
    ('data_segment_size', 'Q'),         # 8 byte size of lengths
    ('offset_to_free_list', 'Q'),       # 8 bytes size of lengths
    ('address_of_data_segment', 'Q'),   # 8 byte addressing
))

# III.E Disk Format: Level 1E - Global Heap
GLOBAL_HEAP_HEADER = OrderedDict((
    ('signature', '4s'),
    ('version', 'B'),
    ('reserved', '3s'),
    ('collection_size', 'Q'),
))
GLOBAL_HEAP_HEADER_SIZE = _structure_size(GLOBAL_HEAP_HEADER)

GLOBAL_HEAP_OBJECT = OrderedDict((
    ('object_index', 'H'),
    ('reference_count', 'H'),
    ('reserved', 'I'),
    ('object_size', 'Q')    # 8 byte addressing
))
GLOBAL_HEAP_OBJECT_SIZE = _structure_size(GLOBAL_HEAP_OBJECT)
<|endoftext|>


<commit_before>import os
import sys
import copier
import image_processor
import gallery_creator

def get_user_path():
    """
    Get the user's destination directory for the gallery folder.

    :return: New gallery directory path as a string.
    """

    dir_name_input = "Gallery path. Created on Desktop. [py_gallery_dist] "
    gallery_dir_name = input(dir_name_input)
    desktop_dir = os.path.join(copier.get_user_desktop(), 'Desktop')
    if gallery_dir_name == "":
        gallery_dir_name = copier.default_dist_path
    else:
        user_def_path = os.path.join(copier.get_user_desktop(), gallery_dir_name)
        gallery_dir_name = os.path.join(user_def_path)
    return gallery_dir_name

def get_raw_image_dir():
    """
    Get the user's image directory.

    :return: User's raw image directory as a string.
    """

    d = input("Please enter the path to your image directory: ")
    return d

def get_directories():
    """Get directories."""

    raw_directory = get_raw_image_dir()
    new_gallery_dir = get_user_path()
    return raw_directory, new_gallery_dir

def create_image_gallery(raw_images, new_gallery_path, kind):
    """Create all the elements of the gallery."""

    copier.make_dist_dir(new_gallery_path)
    copier.copy_resources_gui("vendor", new_gallery_path)
    image_processor.create_images(raw_images, new_gallery_path)
    filename_directory = os.path.join(new_gallery_path, "large_imgs")
    html = gallery_creator.create_html(filename_directory, kind)
    with open(os.path.join(new_gallery_path, "index.html"), "w+") as f:
        f.write(html)

if __name__ == "__main__":
    user_dir, new_gallery = get_directories()
    kind = input("What type of gallery (bootstrap, montage)? ")
    create_image_gallery(user_dir, new_gallery, kind)
<commit_msg>Made bootstrap default kind<commit_after>import os
import sys
import copier
import image_processor
import gallery_creator

def get_user_path():
    """
    Get the user's destination directory for the gallery folder.

    :return: New gallery directory path as a string.
    """

    dir_name_input = "Gallery path. Created on Desktop. [py_gallery_dist] "
    gallery_dir_name = input(dir_name_input)
    desktop_dir = os.path.join(copier.get_user_desktop(), 'Desktop')
    if gallery_dir_name == "":
        gallery_dir_name = copier.default_dist_path
    else:
        user_def_path = os.path.join(copier.get_user_desktop(), gallery_dir_name)
        gallery_dir_name = os.path.join(user_def_path)
    return gallery_dir_name

def get_raw_image_dir():
    """
    Get the user's image directory.

    :return: User's raw image directory as a string.
    """

    d = input("Please enter the path to your image directory: ")
    return d

def get_directories():
    """Get directories."""

    raw_directory = get_raw_image_dir()
    new_gallery_dir = get_user_path()
    return raw_directory, new_gallery_dir

def create_image_gallery(raw_images, new_gallery_path, kind="bootstrap"):
    """Create all the elements of the gallery."""
    if kind == "":
        kind = "bootstrap"
    else:
        pass
    copier.make_dist_dir(new_gallery_path)
    copier.copy_resources_gui("vendor", new_gallery_path)
    image_processor.create_images(raw_images, new_gallery_path)
    filename_directory = os.path.join(new_gallery_path, "large_imgs")
    html = gallery_creator.create_html(filename_directory, kind)
    with open(os.path.join(new_gallery_path, "index.html"), "w+") as f:
        f.write(html)

if __name__ == "__main__":
    user_dir, new_gallery = get_directories()
    kind = input("What type of gallery (bootstrap, montage)? [bootstrap]")
    create_image_gallery(user_dir, new_gallery, kind)
<|endoftext|>


<commit_before>"""
https://oj.leetcode.com/problems/palindrome-number/

Determine whether an integer is a palindrome. Do this without extra space.
"""

class Solution:
    # @return a boolean
    def isPalindrome(self, x):
        if x < 0:
            return False
        else:
            l = Solution.get_len(x)
            for i in range(l / 2):
                fst, x = divmod(x, 10 ** (l - 2 * i - 1))
                x, last = divmod(x, 10)
                if fst != last:
                    return False
            else:
                return True

    @staticmethod
    def get_len(x):
        """Get length of int x."""
        if x == 0:
            return 1
        else:
            l = 0
            while x != 0:
                x /= 10
                l += 1
        return l

if __name__ == '__main__':
    s = Solution()
    assert s.get_len(0) == 1
    assert s.get_len(3) == 1
    assert s.get_len(333) == 3
    assert s.get_len(13254) == 5
    assert s.isPalindrome(0)
    assert s.isPalindrome(1)
    assert s.isPalindrome(131)
    assert s.isPalindrome(1331)
    assert s.isPalindrome(1318131)
    assert not s.isPalindrome(23)
    assert not s.isPalindrome(13254)
    assert not s.isPalindrome(-2147483648)
    assert not s.isPalindrome(-2147447412)
<commit_msg>More clarity.<commit_after>"""
https://oj.leetcode.com/problems/palindrome-number/

Determine whether an integer is a palindrome. Do this without extra space.
"""

class Solution:
    # @return a boolean
    def isPalindrome(self, x):
        if x < 0:
            return False
        else:
            l = Solution.get_len(x)
            for i in range(l / 2):
                fst, x = divmod(x, 10 ** (l - 2 * i - 1))
                x, last = divmod(x, 10)
                if fst != last:
                    return False
            else:
                return True

    @staticmethod
    def get_len(x):
        """Get length of int x."""
        if x == 0:
            return 1
        else:
            l = 0
            while x != 0:
                x /= 10
                l += 1
            return l

if __name__ == '__main__':
    s = Solution()
    assert s.get_len(0) == 1
    assert s.get_len(3) == 1
    assert s.get_len(333) == 3
    assert s.get_len(13254) == 5
    assert s.isPalindrome(0)
    assert s.isPalindrome(1)
    assert s.isPalindrome(131)
    assert s.isPalindrome(1331)
    assert s.isPalindrome(1318131)
    assert not s.isPalindrome(23)
    assert not s.isPalindrome(13254)
    assert not s.isPalindrome(-2147483648)
    assert not s.isPalindrome(-2147447412)
<|endoftext|>


<commit_before>from data_collection.management.commands import BaseXpressWebLookupCsvImporter

class Command(BaseXpressWebLookupCsvImporter):
    council_id = 'E07000004'
    addresses_name = '2017/PropertyPostCodePollingStationWebLookup-2017-05-03.TSV'
    stations_name = '2017/PropertyPostCodePollingStationWebLookup-2017-05-03.TSV'
    elections = [
        'local.buckinghamshire.2017-05-04',
        'parl.2017-06-08'
    ]
    csv_delimiter = '\t'
<commit_msg>update Aylesbury Vale import script<commit_after>from data_collection.management.commands import BaseXpressWebLookupCsvImporter

class Command(BaseXpressWebLookupCsvImporter):
    council_id = 'E07000004'
    addresses_name = 'parl.2017-06-08/Version 1/PropertyPostCodePollingStationWebLookup-2017-05-14.TSV'
    stations_name = 'parl.2017-06-08/Version 1/PropertyPostCodePollingStationWebLookup-2017-05-14.TSV'
    elections = ['parl.2017-06-08']
    csv_delimiter = '\t'
<|endoftext|>


<commit_before>"""
Classify loops through all the ads and save the scores to the database.
"""
import click
import dill
from classifier.utilities import (classifier_path, get_vectorizer,
                                  confs, DB, get_text)

import spacy

# DATABASE_URL="postgres:///facebook_ads" pipenv run ./classify listbuilding_fundraising_classify --every

@click.command("listbuilding_fundraising_classify")
@click.option("--newest/--every",
              default=True,
              help="Classify all of the records")
@click.pass_context
def listbuilding_fundraising_classify(ctx, newest):
    """
    Classify the ads in the database at $DATABASE_URL.
    """
    lang = "en-US" # hard coded
    try:
        nlp = spacy.load('en_fbpac3label') # hard coded
    except OSError as e:
        print("you need to do `pipenv install ~/code/fbpac-prodigy/packagedmodels/en_fbpac3label-2.0.0.tar.gz`")
        raise e

    if newest:
        print("Running newest")
        query = "select * from ads where political_probability > 0.70 and created_at > '2018-08-01' and listbuilding_fundraising_proba is null"
        if lang:
            query = query + " and lang = '{}'".format(lang)
    else:
        print("Running every")
        query = "select * from ads where political_probability > 0.70 and created_at > '2018-08-01'"
        if lang:
            query = query + " and lang = '{}'".format(lang)

    total = "select count(*) as length from ({}) as t1;"
    length = DB.query(total.format(query))[0]["length"]
    records = DB.query(query)
    print("found {} ads".format(length))
    updates = []
    query = "update ads set listbuilding_fundraising_proba=:listbuilding_fundraising_proba where id=:id"
    idx = 0
    for record in records:
        idx += 1
        record_lang = "en-US" if record["lang"] == "en-IE" else record["lang"]

        text = get_text(record).replace("Learn More Watch Again Resume Video Learn More", '')
        doc = nlp(text)
        listbuilding_fundraising_proba = doc.cats["LISTBUILDING"] + doc.cats["FUNDRAISING"]

        update = {
            "id": record["id"],
            "listbuilding_fundraising_proba": listbuilding_fundraising_proba
        }
        updates.append(update)
        out = "Classified {pid[id]} ({info[idx]} of {info[length]}) with {pid[listbuilding_fundraising_proba]}"
        print(out.format(pid=update, info={"length": length, "idx": idx}))

        if len(updates) >= 100:
            DB.bulk_query(query, updates)
            updates = []

    if updates:
        DB.bulk_query(query, updates)
<commit_msg>--every for listbuilding_fundraising_classify should do all the political ads<commit_after>"""
Classify loops through all the ads and save the scores to the database.
"""
import click
import dill
from classifier.utilities import (classifier_path, get_vectorizer,
                                  confs, DB, get_text)

import spacy

# DATABASE_URL="postgres:///facebook_ads" pipenv run ./classify listbuilding_fundraising_classify --every

@click.command("listbuilding_fundraising_classify")
@click.option("--newest/--every",
              default=True,
              help="Classify all of the records")
@click.pass_context
def listbuilding_fundraising_classify(ctx, newest):
    """
    Classify the ads in the database at $DATABASE_URL.
    """
    lang = "en-US" # hard coded
    try:
        nlp = spacy.load('en_fbpac3label') # hard coded
    except OSError as e:
        print("you need to do `pipenv install ~/code/fbpac-prodigy/packagedmodels/en_fbpac3label-2.0.0.tar.gz`")
        raise e

    if newest:
        print("Running newest")
        query = "select * from ads where political_probability > 0.70 and listbuilding_fundraising_proba is null"
        if lang:
            query = query + " and lang = '{}'".format(lang)
    else:
        print("Running every")
        query = "select * from ads where political_probability > 0.70"
        if lang:
            query = query + " and lang = '{}'".format(lang)

    total = "select count(*) as length from ({}) as t1;"
    length = DB.query(total.format(query))[0]["length"]
    records = DB.query(query)
    print("found {} ads".format(length))
    updates = []
    query = "update ads set listbuilding_fundraising_proba=:listbuilding_fundraising_proba where id=:id"
    idx = 0
    for record in records:
        idx += 1
        record_lang = "en-US" if record["lang"] == "en-IE" else record["lang"]

        text = get_text(record).replace("Learn More Watch Again Resume Video Learn More", '')
        doc = nlp(text)
        listbuilding_fundraising_proba = doc.cats["LISTBUILDING"] + doc.cats["FUNDRAISING"]

        update = {
            "id": record["id"],
            "listbuilding_fundraising_proba": listbuilding_fundraising_proba
        }
        updates.append(update)
        out = "Classified {pid[id]} ({info[idx]} of {info[length]}) with {pid[listbuilding_fundraising_proba]}"
        print(out.format(pid=update, info={"length": length, "idx": idx}))

        if len(updates) >= 100:
            DB.bulk_query(query, updates)
            updates = []

    if updates:
        DB.bulk_query(query, updates)
<|endoftext|>


<commit_before>"""Diffbot API wrapper."""
import argparse
import json
import os
import sys
import urllib
import urllib2

try:
    import requests
except ImportError:
    pass

ENCODING = 'utf-8'

API_ROOT = 'http://api.diffbot.com'
API_VERSION = 3

class Client(object):
    """Diffbot client."""

    _apis = frozenset(('article', 'frontpage', 'product', 'image', 'analyze',
                       'discussion'))

    def __init__(self, token, version=API_VERSION):
        """Initialise the client."""
        self._token = token
        self._version = version

    @staticmethod
    def _get(url, params=None):
        """HTTP GET request."""
        try:
            response = requests.get(url, params=params)
            response.raise_for_status()
            # If JSON fails, return raw data
            # (e.g. when downloading CSV job logs).
            try:
                return response.json()
            except ValueError:
                return response.text
        except NameError:
            url = '{0}?{1}'.format(url, urllib.urlencode(params))
            return json.loads(urllib2.urlopen(url).read().decode(ENCODING))

    @staticmethod
    def _post(url, data, content_type, params=None):
        """HTTP POST request."""
        try:
            response = requests.post(url, params=params, data=data, headers={
                'Content-Type': content_type,
            })
            response.raise_for_status()
            return response.json()
        except NameError:
            url = '{0}?{1}'.format(url, urllib.urlencode(params))
            req = urllib2.Request(url, data.encode(ENCODING), {
                'Content-Type': content_type,
            })
            return json.loads(urllib2.urlopen(req).read().decode(ENCODING))

    def endpoint(self, name):
        return '{0}/v{1}/{2}'.format(API_ROOT, self._version, name)

    def api(self, name, url, **kwargs):
        """Generic API method."""
        if name not in self._apis:
            raise ValueError('API name must be one of {0}, not {1!r}.'.format(
                tuple(self._apis), name))
        fields = kwargs.get('fields')
        timeout = kwargs.get('timeout')
        text = kwargs.get('text')
        html = kwargs.get('html')
        if text and html:
            raise ValueError(u'Both `text` and `html` arguments provided!')
        params = {'url': url, 'token': self._token}
        if timeout:
            params['timeout'] = timeout
        if fields:
            if not isinstance(fields, str):
                fields = ','.join(sorted(fields))
            params['fields'] = fields
        url = self.endpoint(name)
        if text or html:
            content_type = html and 'text/html' or 'text/plain'
            return self._post(url, text or html, content_type, params=params)
        return self._get(url, params=params)

    def article(self, url, **kwargs):
        """Article API."""
        return self.api('article', url, **kwargs)

    def frontpage(self, url, **kwargs):
        """Frontpage API."""
        return self.api('frontpage', url, **kwargs)

    def product(self, url, **kwargs):
        """Product API."""
        return self.api('product', url, **kwargs)

    def image(self, url, **kwargs):
        """Image API."""
        return self.api('image', url, **kwargs)

    def analyze(self, url, **kwargs):
        """Classifier (analyze) API."""
        return self.api('analyze', url, **kwargs)

    def discussion(self, url, **kwargs):
        """Discussion API."""
        return self.api('discussion', url, **kwargs)

    def crawl(self, urls, name='crawl', api='analyze', **kwargs):
        """Crawlbot API.

        Returns a diffbot.Job object to check and retrieve crawl status.
        """
        # If multiple seed URLs are specified, join with whitespace.
        if isinstance(urls == list):
            urls = ' '.join(urls)
        url = self.endpoint('crawl')
        process_url = self.endpoint(api)
        params = {
            'token': self._token,
            'seeds': urls,
            'name': name,
            'apiUrl': process_url,
        }

        # Add any additional named parameters as accepted by Crawlbot
        params['maxToCrawl'] = 10
        params.update(kwargs)

        rv = self._get(url, params=params)
        job = next(j for j in rv['jobs'] if j['name'] == name)
        return Job(self._token, name, self._version)

class Job (Client):
    """An asynchronous job.

    This is used to check crawl status once a crawl job was started.
    """

    def __init__(self, token, name, version=API_VERSION):
        Client.__init__(self, token, version)
        self._name = name
        self._url = self.endpoint('crawl')

    def control(self, **kwargs):
        params = {'token': self._token, 'name': self._name}
        params.update(kwargs)
        rv = self._get(self._url, params)
        job = next(j for j in rv['jobs'] if j['name'] == self._name)
        return job

    def pause(self):
        return self.control(pause=1)

    def unpause(self):
        return self.control(pause=0)

    def restart(self):
        return self.control(restart=1)

    def delete(self):
        return self.control(delete=1)

    def status(self):
        return self.control()

    def status_code(self):
        response = self.status()
        return response['jobStatus']['status']

    def is_finished(self):
        status_code = self.status_code()
        finished_codes = [1, 2, 3, 4, 5, 9]
        return status_code in finished_codes

    def is_running(self):
        status_code = self.status_code()
        running_codes = [0, 6, 7, 8]
        return status_code in running_codes

    def download(self, format='json'):
        download_url = '{0}/download/{1}-{2}_data.{3}'\
            .format(self._url, self._token, self._name, format)
        return self._get(download_url)

def api(name, url, token, **kwargs):
    """Shortcut for caling methods on `Client(token, version)`."""
    return Client(token).api(name, url, **kwargs)

def article(url, token, **kwargs):
    """Shortcut for `Client(token, version).article(url)`."""
    return api('article', url, token, **kwargs)

def frontpage(url, token, **kwargs):
    """Shortcut for `Client(token, version).frontpage(url)`."""
    return api('frontpage', url, token, **kwargs)

def product(url, token, **kwargs):
    """Shortcut for `Client(token, version).product(url)`."""
    return api('product', url, token, **kwargs)

def image(url, token, **kwargs):
    """Shortcut for `Client(token, version).image(url)`."""
    return api('image', url, token, **kwargs)

def analyze(url, token, **kwargs):
    """Shortcut for `Client(token, version).analyze(url)`."""
    return api('analyze', url, token, **kwargs)

def discussion(url, token, **kwargs):
    """Shortcut for `Client(token, version).discussion(url)`."""
    return api('discussion', url, token, **kwargs)

def cli():
    """Command line tool."""
    parser = argparse.ArgumentParser()
    parser.add_argument('api', help="""
        API to call.
        One one of 'article', 'frontpage', 'product', 'image', 'analyze', or
        'discussion'.
    """)
    parser.add_argument('url', help="""
        URL to pass as the 'url' parameter.
    """)
    parser.add_argument('token', help="""
        API key (token).
        Get one at https://www.diffbot.com/.
    """)
    parser.add_argument('-a', '--all', help="""
        Request all fields.
    """, action='store_true')
    parser.add_argument('-f', '--file', help="""
        File to read data from.
        Use '-' to read from STDIN.
    """)
    fields = text = html = None
    _args = parser.parse_args()
    if _args.all:
        fields = '*'
    if _args.file == '-':
        text = sys.stdin.read()
    elif _args.file:
        with open(_args.file, 'rb') as src:
            if os.path.splitext(_args.file)[1] in ('.html', '.htm'):
                html = src.read().decode(ENCODING)
            else:
                text = src.read().decode(ENCODING)
    print(json.dumps((api(_args.api, _args.url, _args.token,
                          html=html or None,
                          text=text or None,
                          fields=fields)),
                     sort_keys=True,
                     indent=2))

if __name__ == '__main__':
    cli()  # pragma: no cover
<commit_msg>more PEP8, remove unused variable declaration<commit_after>"""Diffbot API wrapper."""
import argparse
import json
import os
import sys
import urllib
import urllib2

try:
    import requests
except ImportError:
    pass

ENCODING = 'utf-8'

API_ROOT = 'http://api.diffbot.com'
API_VERSION = 3

class Client(object):
    """Diffbot client."""

    _apis = frozenset(('article', 'frontpage', 'product', 'image', 'analyze',
                       'discussion'))

    def __init__(self, token, version=API_VERSION):
        """Initialise the client."""
        self._token = token
        self._version = version

    @staticmethod
    def _get(url, params=None):
        """HTTP GET request."""
        try:
            response = requests.get(url, params=params)
            response.raise_for_status()
            # If JSON fails, return raw data
            # (e.g. when downloading CSV job logs).
            try:
                return response.json()
            except ValueError:
                return response.text
        except NameError:
            url = '{0}?{1}'.format(url, urllib.urlencode(params))
            return json.loads(urllib2.urlopen(url).read().decode(ENCODING))

    @staticmethod
    def _post(url, data, content_type, params=None):
        """HTTP POST request."""
        try:
            response = requests.post(url, params=params, data=data, headers={
                'Content-Type': content_type,
            })
            response.raise_for_status()
            return response.json()
        except NameError:
            url = '{0}?{1}'.format(url, urllib.urlencode(params))
            req = urllib2.Request(url, data.encode(ENCODING), {
                'Content-Type': content_type,
            })
            return json.loads(urllib2.urlopen(req).read().decode(ENCODING))

    def endpoint(self, name):
        """Generate the URL endpoint for the given API."""
        return '{0}/v{1}/{2}'.format(API_ROOT, self._version, name)

    def api(self, name, url, **kwargs):
        """Generic API method."""
        if name not in self._apis:
            raise ValueError('API name must be one of {0}, not {1!r}.'.format(
                tuple(self._apis), name))
        fields = kwargs.get('fields')
        timeout = kwargs.get('timeout')
        text = kwargs.get('text')
        html = kwargs.get('html')
        if text and html:
            raise ValueError(u'Both `text` and `html` arguments provided!')
        params = {'url': url, 'token': self._token}
        if timeout:
            params['timeout'] = timeout
        if fields:
            if not isinstance(fields, str):
                fields = ','.join(sorted(fields))
            params['fields'] = fields
        url = self.endpoint(name)
        if text or html:
            content_type = html and 'text/html' or 'text/plain'
            return self._post(url, text or html, content_type, params=params)
        return self._get(url, params=params)

    def article(self, url, **kwargs):
        """Article API."""
        return self.api('article', url, **kwargs)

    def frontpage(self, url, **kwargs):
        """Frontpage API."""
        return self.api('frontpage', url, **kwargs)

    def product(self, url, **kwargs):
        """Product API."""
        return self.api('product', url, **kwargs)

    def image(self, url, **kwargs):
        """Image API."""
        return self.api('image', url, **kwargs)

    def analyze(self, url, **kwargs):
        """Classifier (analyze) API."""
        return self.api('analyze', url, **kwargs)

    def discussion(self, url, **kwargs):
        """Discussion API."""
        return self.api('discussion', url, **kwargs)

    def crawl(self, urls, name='crawl', api='analyze', **kwargs):
        """Crawlbot API.

        Returns a diffbot.Job object to check and retrieve crawl status.
        """
        # If multiple seed URLs are specified, join with whitespace.
        if isinstance(urls == list):
            urls = ' '.join(urls)
        url = self.endpoint('crawl')
        process_url = self.endpoint(api)
        params = {
            'token': self._token,
            'seeds': urls,
            'name': name,
            'apiUrl': process_url,
        }

        # Add any additional named parameters as accepted by Crawlbot
        params['maxToCrawl'] = 10
        params.update(kwargs)

        self._get(url, params=params)

        return Job(self._token, name, self._version)

class Job(Client):
    """An asynchronous job.

    This is used to check crawl status once a crawl job was started.
    """

    def __init__(self, token, name, version=API_VERSION):
        Client.__init__(self, token, version)
        self._name = name
        self._url = self.endpoint('crawl')

    def control(self, **kwargs):
        params = {'token': self._token, 'name': self._name}
        params.update(kwargs)
        res = self._get(self._url, params)
        job = next(j for j in res['jobs'] if j['name'] == self._name)
        return job

    def pause(self):
        return self.control(pause=1)

    def unpause(self):
        return self.control(pause=0)

    def restart(self):
        return self.control(restart=1)

    def delete(self):
        return self.control(delete=1)

    def status(self):
        return self.control()

    def status_code(self):
        response = self.status()
        return response['jobStatus']['status']

    def is_finished(self):
        status_code = self.status_code()
        finished_codes = [1, 2, 3, 4, 5, 9]
        return status_code in finished_codes

    def is_running(self):
        status_code = self.status_code()
        running_codes = [0, 6, 7, 8]
        return status_code in running_codes

    def download(self, format='json'):
        download_url = '{0}/download/{1}-{2}_data.{3}'\
            .format(self._url, self._token, self._name, format)
        return self._get(download_url)

def api(name, url, token, **kwargs):
    """Shortcut for caling methods on `Client(token, version)`."""
    return Client(token).api(name, url, **kwargs)

def article(url, token, **kwargs):
    """Shortcut for `Client(token, version).article(url)`."""
    return api('article', url, token, **kwargs)

def frontpage(url, token, **kwargs):
    """Shortcut for `Client(token, version).frontpage(url)`."""
    return api('frontpage', url, token, **kwargs)

def product(url, token, **kwargs):
    """Shortcut for `Client(token, version).product(url)`."""
    return api('product', url, token, **kwargs)

def image(url, token, **kwargs):
    """Shortcut for `Client(token, version).image(url)`."""
    return api('image', url, token, **kwargs)

def analyze(url, token, **kwargs):
    """Shortcut for `Client(token, version).analyze(url)`."""
    return api('analyze', url, token, **kwargs)

def discussion(url, token, **kwargs):
    """Shortcut for `Client(token, version).discussion(url)`."""
    return api('discussion', url, token, **kwargs)

def cli():
    """Command line tool."""
    parser = argparse.ArgumentParser()
    parser.add_argument('api', help="""
        API to call.
        One one of 'article', 'frontpage', 'product', 'image', 'analyze', or
        'discussion'.
    """)
    parser.add_argument('url', help="""
        URL to pass as the 'url' parameter.
    """)
    parser.add_argument('token', help="""
        API key (token).
        Get one at https://www.diffbot.com/.
    """)
    parser.add_argument('-a', '--all', help="""
        Request all fields.
    """, action='store_true')
    parser.add_argument('-f', '--file', help="""
        File to read data from.
        Use '-' to read from STDIN.
    """)
    fields = text = html = None
    _args = parser.parse_args()
    if _args.all:
        fields = '*'
    if _args.file == '-':
        text = sys.stdin.read()
    elif _args.file:
        with open(_args.file, 'rb') as src:
            if os.path.splitext(_args.file)[1] in ('.html', '.htm'):
                html = src.read().decode(ENCODING)
            else:
                text = src.read().decode(ENCODING)
    print(json.dumps((api(_args.api, _args.url, _args.token,
                          html=html or None,
                          text=text or None,
                          fields=fields)),
                     sort_keys=True,
                     indent=2))

if __name__ == '__main__':
    cli()  # pragma: no cover
<|endoftext|>


<commit_before>from rsqueakvm import constants
from rsqueakvm.error import PrimitiveFailedError, MetaPrimFailed
from rsqueakvm.model.numeric import W_SmallInteger
from rsqueakvm.primitives import expose_primitive
from rsqueakvm.primitives.bytecodes import *

# ___________________________________________________________________________
# Drawing

@expose_primitive(IDLE_FOR_MICROSECONDS,
                  unwrap_spec=[object, int], no_result=True, clean_stack=False)
def func(interp, s_frame, w_rcvr, time_mu_s):
    import time
    s_frame.pop()
    time_s = time_mu_s / 1000000.0
    interp.interrupt_check_counter = 0
    interp.quick_check_for_interrupt(s_frame, dec=0)
    time.sleep(time_s)
    interp.interrupt_check_counter = 0
    interp.quick_check_for_interrupt(s_frame, dec=0)

# @expose_primitive(FORCE_DISPLAY_UPDATE, unwrap_spec=[object])
# def func(interp, s_frame, w_rcvr):
#     interp.space.display().flip(force=True)
#     return w_rcvr

@expose_primitive(SET_FULL_SCREEN, unwrap_spec=[object, bool])
def func(interp, s_frame, w_rcvr, flag):
    interp.space.display().set_full_screen(flag)
    return w_rcvr

# ___________________________________________________________________________
# VM implementor primitives

@expose_primitive(META_PRIM_FAILED, unwrap_spec=[object, int])
def func(interp, s_frame, w_rcvr, primFailFlag):
    if primFailFlag != 0:
        raise MetaPrimFailed(s_frame, primFailFlag)
    raise PrimitiveFailedError

@expose_primitive(VM_PARAMETERS)
def func(interp, s_frame, argcount):
    from rpython.rlib import jit_hooks
    """Behaviour depends on argument count:
            0 args: return an Array of VM parameter values;
            1 arg:  return the indicated VM parameter;
            2 args: set the VM indicated parameter.
        VM parameters are numbered as follows:
            1   byte size of old-space (read-only)
            2   byte size of young-space (read-only)
            3   byte size of object memory (read-only)
            4   allocationCount (read-only; nil in Cog VMs)
            5   allocations between GCs (read-write; nil in Cog VMs)
            6   survivor count tenuring threshold (read-write)
            7   full GCs since startup (read-only)
            8   total milliseconds in full GCs since startup (read-only)
            9   incremental GCs since startup (read-only; scavenging GCs on Spur)
            10  total milliseconds in incremental/scavenging GCs since startup (read-only)
            11  tenures of surving objects since startup (read-only)
            12-20 specific to the translating VM (nil in Cog VMs)
            21  root table size (read-only)
            22  root table overflows since startup (read-only)
            23  bytes of extra memory to reserve for VM buffers, plugins, etc.
            24  memory threshold above which to shrink object memory (read-write)
            25  ammount to grow by when growing object memory (read-write)
            26  interruptChecksEveryNms - force an ioProcessEvents every N milliseconds (read-write)
            27  number of times mark loop iterated for current IGC/FGC (read-only) includes ALL marking
            28  number of times sweep loop iterated for current IGC/FGC (read-only)
            29  number of times make forward loop iterated for current IGC/FGC (read-only)
            30  number of times compact move loop iterated for current IGC/FGC (read-only)
            31  number of grow memory requests (read-only)
            32  number of shrink memory requests (read-only)
            33  number of root table entries used for current IGC/FGC (read-only)
            34  number of allocations done before current IGC/FGC (read-only)
            35  number of survivor objects after current IGC/FGC (read-only)
            36  millisecond clock when current IGC/FGC completed (read-only)
            37  number of marked objects for Roots of the world, not including Root Table entries for current IGC/FGC (read-only)
            38  milliseconds taken by current IGC (read-only)
            39  Number of finalization signals for Weak Objects pending when current IGC/FGC completed (read-only)
            40  BytesPerWord for this image
            41  imageFormatVersion for the VM
            42  number of stack pages in use (Cog Stack VM only, otherwise nil)
            43  desired number of stack pages (stored in image file header, max 65535; Cog VMs only, otherwise nil)
            44  size of eden, in bytes (Cog VMs only, otherwise nil)
            45  desired size of eden, in bytes (stored in image file header; Cog VMs only, otherwise nil)
            46  size of machine code zone, in bytes (stored in image file header; Cog JIT VM only, otherwise nil)
            47  desired size of machine code zone, in bytes (applies at startup only, stored in image file header; Cog JIT VM only)
            48  various properties of the Cog VM as an integer encoding an array of bit flags.
                Bit 0: implies the image's Process class has threadId as its 3rd inst var (zero relative)
                Bit 1: on Cog VMs asks the VM to set the flag bit in interpreted methods
                Bit 2: if set, preempting a process puts it to the head of its run queue, not the back,
                        i.e. preempting a process by a higher one will not cause the process to yield
                            to others at the same priority.
            49  the size of the external semaphore table (read-write; Cog VMs only)
            50-53 reserved for VM parameters that persist in the image (such as eden above)
            54  total size of free old space (Spur only, otherwise nil)
            55  ratio of growth and image size at or above which a GC will be performed post scavenge (Spur only, otherwise nil)
            56  number of process switches since startup (read-only)
            57  number of ioProcessEvents calls since startup (read-only)
            58  number of forceInterruptCheck (Cog VMs) or quickCheckInterruptCalls (non-Cog VMs) calls since startup (read-only)
            59  number of check event calls since startup (read-only)
            60  number of stack page overflows since startup (read-only; Cog VMs only)
            61  number of stack page divorces since startup (read-only; Cog VMs only)
            62  number of machine code zone compactions since startup (read-only; Cog VMs only)
            63  milliseconds taken by machine code zone compactions since startup (read-only; Cog VMs only)
            64  current number of machine code methods (read-only; Cog VMs only)
            65  true if the VM supports multiple bytecode sets;  (read-only; Cog VMs only; nil in older Cog VMs)
            66  the byte size of a stack page in the stack zone  (read-only; Cog VMs only)
            67 - 69 reserved for more Cog-related info
            70  the value of VM_PROXY_MAJOR (the interpreterProxy major version number)
            71  the value of VM_PROXY_MINOR (the interpreterProxy minor version number)

        Note: Thanks to Ian Piumarta for this primitive."""

    if not 0 <= argcount <= 2:
        raise PrimitiveFailedError

    arg1_w = s_frame.pop()  # receiver

    vm_w_params = [interp.space.wrap_int(0)] * 71

    vm_w_params[2] = interp.space.wrap_int(1)  # must be 1 for VM Stats view to work
    vm_w_params[8] = interp.space.wrap_int(1)  # must be 1 for VM Stats view to work

    vm_w_params[41] = interp.space.wrap_int(1)  # We are a "stack-like" VM - number of stack tables
    # sizeOfMachineCode = jit_hooks.stats_asmmemmgr_allocated(None)
    sizeOfMachineCode = jit_hooks.stats_asmmemmgr_used(None)
    vm_w_params[45] = interp.space.wrap_int(sizeOfMachineCode)

    vm_w_params[39] = interp.space.wrap_int(constants.BYTES_PER_WORD)
    vm_w_params[40] = interp.space.wrap_int(interp.image.version.magic)
    vm_w_params[55] = interp.space.wrap_int(interp.process_switch_count)
    vm_w_params[57] = interp.space.wrap_int(interp.forced_interrupt_checks_count)
    vm_w_params[59] = interp.space.wrap_int(interp.stack_overflow_count)

    numberOfLoops = jit_hooks.stats_get_counter_value("TOTAL_COMPILED_LOOPS")
    numberOfBridges = jit_hooks.stats_get_counter_value("TOTAL_COMPILED_BRIDGES")
    vm_w_params[63] = interp.space.wrap_int(numberOfLoops + numberOfBridges)

    vm_w_params[69] = interp.space.wrap_int(constants.INTERP_PROXY_MAJOR)
    vm_w_params[70] = interp.space.wrap_int(constants.INTERP_PROXY_MINOR)

    if argcount == 0:
        return interp.space.wrap_list(vm_w_params)

    arg2_w = s_frame.pop()  # index (really the receiver, index has been removed above)
    if not isinstance(arg1_w, W_SmallInteger):
        raise PrimitiveFailedError
    if argcount == 1:
        if not 0 <= arg1_w.value <= 70:
            raise PrimitiveFailedError
        return vm_w_params[arg1_w.value - 1]

    s_frame.pop()  # new value
    if argcount == 2:
        # return the 'old value'
        return interp.space.wrap_int(0)

# list the n-th loaded module
@expose_primitive(VM_LOADED_MODULES, unwrap_spec=[int])
def func(interp, s_frame, index):
    if interp.space.use_plugins.is_set():
        from rsqueakvm.plugins.squeak_plugin_proxy import IProxy
        modulenames = IProxy.loaded_module_names()
        try:
            return interp.space.wrap_string(modulenames[index])
        except IndexError:
            return interp.space.w_nil
    return interp.space.w_nil
<commit_msg>fix translation for jit stats<commit_after>from rpython.rlib import jit

from rsqueakvm import constants
from rsqueakvm.error import PrimitiveFailedError, MetaPrimFailed
from rsqueakvm.model.numeric import W_SmallInteger
from rsqueakvm.primitives import expose_primitive
from rsqueakvm.primitives.bytecodes import *

# ___________________________________________________________________________
# Drawing

@expose_primitive(IDLE_FOR_MICROSECONDS,
                  unwrap_spec=[object, int], no_result=True, clean_stack=False)
def func(interp, s_frame, w_rcvr, time_mu_s):
    import time
    s_frame.pop()
    time_s = time_mu_s / 1000000.0
    interp.interrupt_check_counter = 0
    interp.quick_check_for_interrupt(s_frame, dec=0)
    time.sleep(time_s)
    interp.interrupt_check_counter = 0
    interp.quick_check_for_interrupt(s_frame, dec=0)

# @expose_primitive(FORCE_DISPLAY_UPDATE, unwrap_spec=[object])
# def func(interp, s_frame, w_rcvr):
#     interp.space.display().flip(force=True)
#     return w_rcvr

@expose_primitive(SET_FULL_SCREEN, unwrap_spec=[object, bool])
def func(interp, s_frame, w_rcvr, flag):
    interp.space.display().set_full_screen(flag)
    return w_rcvr

# ___________________________________________________________________________
# VM implementor primitives

@expose_primitive(META_PRIM_FAILED, unwrap_spec=[object, int])
def func(interp, s_frame, w_rcvr, primFailFlag):
    if primFailFlag != 0:
        raise MetaPrimFailed(s_frame, primFailFlag)
    raise PrimitiveFailedError

@expose_primitive(VM_PARAMETERS)
@jit.dont_look_inside
def func(interp, s_frame, argcount):
    from rpython.rlib import jit_hooks
    """Behaviour depends on argument count:
            0 args: return an Array of VM parameter values;
            1 arg:  return the indicated VM parameter;
            2 args: set the VM indicated parameter.
        VM parameters are numbered as follows:
            1   byte size of old-space (read-only)
            2   byte size of young-space (read-only)
            3   byte size of object memory (read-only)
            4   allocationCount (read-only; nil in Cog VMs)
            5   allocations between GCs (read-write; nil in Cog VMs)
            6   survivor count tenuring threshold (read-write)
            7   full GCs since startup (read-only)
            8   total milliseconds in full GCs since startup (read-only)
            9   incremental GCs since startup (read-only; scavenging GCs on Spur)
            10  total milliseconds in incremental/scavenging GCs since startup (read-only)
            11  tenures of surving objects since startup (read-only)
            12-20 specific to the translating VM (nil in Cog VMs)
            21  root table size (read-only)
            22  root table overflows since startup (read-only)
            23  bytes of extra memory to reserve for VM buffers, plugins, etc.
            24  memory threshold above which to shrink object memory (read-write)
            25  ammount to grow by when growing object memory (read-write)
            26  interruptChecksEveryNms - force an ioProcessEvents every N milliseconds (read-write)
            27  number of times mark loop iterated for current IGC/FGC (read-only) includes ALL marking
            28  number of times sweep loop iterated for current IGC/FGC (read-only)
            29  number of times make forward loop iterated for current IGC/FGC (read-only)
            30  number of times compact move loop iterated for current IGC/FGC (read-only)
            31  number of grow memory requests (read-only)
            32  number of shrink memory requests (read-only)
            33  number of root table entries used for current IGC/FGC (read-only)
            34  number of allocations done before current IGC/FGC (read-only)
            35  number of survivor objects after current IGC/FGC (read-only)
            36  millisecond clock when current IGC/FGC completed (read-only)
            37  number of marked objects for Roots of the world, not including Root Table entries for current IGC/FGC (read-only)
            38  milliseconds taken by current IGC (read-only)
            39  Number of finalization signals for Weak Objects pending when current IGC/FGC completed (read-only)
            40  BytesPerWord for this image
            41  imageFormatVersion for the VM
            42  number of stack pages in use (Cog Stack VM only, otherwise nil)
            43  desired number of stack pages (stored in image file header, max 65535; Cog VMs only, otherwise nil)
            44  size of eden, in bytes (Cog VMs only, otherwise nil)
            45  desired size of eden, in bytes (stored in image file header; Cog VMs only, otherwise nil)
            46  size of machine code zone, in bytes (stored in image file header; Cog JIT VM only, otherwise nil)
            47  desired size of machine code zone, in bytes (applies at startup only, stored in image file header; Cog JIT VM only)
            48  various properties of the Cog VM as an integer encoding an array of bit flags.
                Bit 0: implies the image's Process class has threadId as its 3rd inst var (zero relative)
                Bit 1: on Cog VMs asks the VM to set the flag bit in interpreted methods
                Bit 2: if set, preempting a process puts it to the head of its run queue, not the back,
                        i.e. preempting a process by a higher one will not cause the process to yield
                            to others at the same priority.
            49  the size of the external semaphore table (read-write; Cog VMs only)
            50-53 reserved for VM parameters that persist in the image (such as eden above)
            54  total size of free old space (Spur only, otherwise nil)
            55  ratio of growth and image size at or above which a GC will be performed post scavenge (Spur only, otherwise nil)
            56  number of process switches since startup (read-only)
            57  number of ioProcessEvents calls since startup (read-only)
            58  number of forceInterruptCheck (Cog VMs) or quickCheckInterruptCalls (non-Cog VMs) calls since startup (read-only)
            59  number of check event calls since startup (read-only)
            60  number of stack page overflows since startup (read-only; Cog VMs only)
            61  number of stack page divorces since startup (read-only; Cog VMs only)
            62  number of machine code zone compactions since startup (read-only; Cog VMs only)
            63  milliseconds taken by machine code zone compactions since startup (read-only; Cog VMs only)
            64  current number of machine code methods (read-only; Cog VMs only)
            65  true if the VM supports multiple bytecode sets;  (read-only; Cog VMs only; nil in older Cog VMs)
            66  the byte size of a stack page in the stack zone  (read-only; Cog VMs only)
            67 - 69 reserved for more Cog-related info
            70  the value of VM_PROXY_MAJOR (the interpreterProxy major version number)
            71  the value of VM_PROXY_MINOR (the interpreterProxy minor version number)

        Note: Thanks to Ian Piumarta for this primitive."""

    if not 0 <= argcount <= 2:
        raise PrimitiveFailedError

    arg1_w = s_frame.pop()  # receiver

    vm_w_params = [interp.space.wrap_int(0)] * 71

    vm_w_params[2] = interp.space.wrap_int(1)  # must be 1 for VM Stats view to work
    vm_w_params[8] = interp.space.wrap_int(1)  # must be 1 for VM Stats view to work

    vm_w_params[41] = interp.space.wrap_int(1)  # We are a "stack-like" VM - number of stack tables
    # sizeOfMachineCode = jit_hooks.stats_asmmemmgr_allocated(None)
    sizeOfMachineCode = jit_hooks.stats_asmmemmgr_used(None)
    vm_w_params[45] = interp.space.wrap_int(sizeOfMachineCode)

    vm_w_params[39] = interp.space.wrap_int(constants.BYTES_PER_WORD)
    vm_w_params[40] = interp.space.wrap_int(interp.image.version.magic)
    vm_w_params[55] = interp.space.wrap_int(interp.process_switch_count)
    vm_w_params[57] = interp.space.wrap_int(interp.forced_interrupt_checks_count)
    vm_w_params[59] = interp.space.wrap_int(interp.stack_overflow_count)

    numberOfLoops = jit_hooks.stats_get_counter_value(None, jit.Counters.TOTAL_COMPILED_LOOPS)
    numberOfBridges = jit_hooks.stats_get_counter_value(None, jit.Counters.TOTAL_COMPILED_BRIDGES)
    vm_w_params[63] = interp.space.wrap_int(numberOfLoops + numberOfBridges)

    vm_w_params[69] = interp.space.wrap_int(constants.INTERP_PROXY_MAJOR)
    vm_w_params[70] = interp.space.wrap_int(constants.INTERP_PROXY_MINOR)

    if argcount == 0:
        return interp.space.wrap_list(vm_w_params)

    arg2_w = s_frame.pop()  # index (really the receiver, index has been removed above)
    if not isinstance(arg1_w, W_SmallInteger):
        raise PrimitiveFailedError
    if argcount == 1:
        if not 0 <= arg1_w.value <= 70:
            raise PrimitiveFailedError
        return vm_w_params[arg1_w.value - 1]

    s_frame.pop()  # new value
    if argcount == 2:
        # return the 'old value'
        return interp.space.wrap_int(0)

# list the n-th loaded module
@expose_primitive(VM_LOADED_MODULES, unwrap_spec=[int])
def func(interp, s_frame, index):
    if interp.space.use_plugins.is_set():
        from rsqueakvm.plugins.squeak_plugin_proxy import IProxy
        modulenames = IProxy.loaded_module_names()
        try:
            return interp.space.wrap_string(modulenames[index])
        except IndexError:
            return interp.space.w_nil
    return interp.space.w_nil
<|endoftext|>


<commit_before>###############################################################################
## Copyright (c) 2013-2014, Intel Corporation                                ##
## All rights reserved.                                                      ##
##                                                                           ##
## Redistribution and use in source and binary forms, with or without        ##
## modification, are permitted provided that the following conditions        ##
## are met:                                                                  ##
## 1. Redistributions of source code must retain the above copyright         ##
##    notice, this list of conditions and the following disclaimer.          ##
## 2. Redistributions in binary form must reproduce the above copyright      ##
##    notice, this list of conditions and the following disclaimer in the    ##
##    documentation and/or other materials provided with the distribution.   ##
## 3. Neither the name of the copyright holder nor the names of its          ##
##    contributors may be used to endorse or promote products derived        ##
##    from this software without specific prior written permission.          ##
##                                                                           ##
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS       ##
## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT         ##
## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR     ##
## A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT      ##
## HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,    ##
## SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED  ##
## TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR    ##
## PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF    ##
## LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING      ##
## NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS        ##
## SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.              ##
###############################################################################
## Christopher Dahnken (Intel Corp.), Hans Pabst (Intel Corp.),
## Alfio Lazzaro (CRAY Inc.), and Gilles Fourestey (CSCS)
###############################################################################
import math
import sys

def create_macros(RowMajor, maxMNK):
    print "#define LIBXSMM_MAX_MNK " + str(maxMNK)
    if (0 != RowMajor):
        print "#define LIBXSMM_ROW_MAJOR 1"
        print "#define LIBXSMM_COL_MAJOR 0"
        print
        print "#define LIBXSMM_BLASMM(REAL, UINT, M, N, K, A, B, C) { \\"
        print "  REAL libxsmm_alpha_ = 1, libxsmm_beta_ = 1; \\"
        print "  UINT libxsmm_m_ = (M), libxsmm_n_ = (N), libxsmm_k_ = (K); \\"
        print "  char libxsmm_trans_ = 'N'; \\"
        print "  LIBXSMM_BLASPREC(, REAL, gemm)(&libxsmm_trans_, &libxsmm_trans_, \\"
        print "    &libxsmm_n_, &libxsmm_m_, &libxsmm_k_, \\"
        print "    &libxsmm_alpha_, (REAL*)(B), &libxsmm_n_, (REAL*)(A), &libxsmm_k_, \\"
        print "    &libxsmm_beta_, (C), &libxsmm_n_); \\"
        print "}"
    else:
        print "#define LIBXSMM_ROW_MAJOR 0"
        print "#define LIBXSMM_COL_MAJOR 1"
        print
        print "#define LIBXSMM_BLASMM(REAL, UINT, M, N, K, A, B, C) { \\"
        print "  UINT libxsmm_m_ = (M), libxsmm_n_ = (N), libxsmm_k_ = (K); \\"
        print "  REAL libxsmm_alpha_ = 1, libxsmm_beta_ = 1; \\"
        print "  char libxsmm_trans_ = 'N'; \\"
        print "  LIBXSMM_BLASPREC(, REAL, gemm)(&libxsmm_trans_, &libxsmm_trans_, \\"
        print "    &libxsmm_m_, &libxsmm_n_, &libxsmm_k_, \\"
        print "    &libxsmm_alpha_, (REAL*)(A), &libxsmm_k_, (REAL*)(B), &libxsmm_n_, \\"
        print "    &libxsmm_beta_, (C), &libxsmm_n_); \\"
        print "}"
    print
    print "#if defined(MKL_DIRECT_CALL_SEQ) || defined(MKL_DIRECT_CALL)"
    print "# define LIBXSMM_SMM(REAL, UINT, M, N, K, A, B, C) LIBXSMM_BLASMM(REAL, UINT, M, N, K, A, B, C)"
    print "#else"
    if (0 != RowMajor):
        print "# define LIBXSMM_SMM(REAL, UINT, M, N, K, A, B, C) { \\"
        print "    UINT libxsmm_i_, libxsmm_j_, libxsmm_k_; \\"
        print "    REAL *const libxsmm_c_ = (C); \\"
        print "    LIBXSMM_PRAGMA(vector nontemporal(libxsmm_c_)) \\"
        print "    LIBXSMM_PRAGMA(/*omp*/ simd collapse(2)) \\"
        print "    for (libxsmm_j_ = 0; libxsmm_j_ < (N); ++libxsmm_j_) { \\"
        print "      for (libxsmm_i_ = 0; libxsmm_i_ < (M); ++libxsmm_i_) { \\"
        print "        const UINT libxsmm_index_ = libxsmm_i_ * (N) + libxsmm_j_; \\"
        print "        REAL libxsmm_r_ = libxsmm_c_[libxsmm_index_]; \\"
        print "        LIBXSMM_PRAGMA(unroll(16)) \\"
        print "        LIBXSMM_PRAGMA(/*omp*/ simd reduction(+:libxsmm_r_)) \\"
        print "        for (libxsmm_k_ = 0; libxsmm_k_ < (K); ++libxsmm_k_) { \\"
        print "          libxsmm_r_ += (A)[libxsmm_i_*K+libxsmm_k_] * (B)[libxsmm_k_*N+libxsmm_j_]; \\"
        print "        } \\"
        print "        libxsmm_c_[libxsmm_index_] = libxsmm_r_; \\"
        print "      } \\"
        print "    } \\"
        print "  }"
    else:
        print "# define LIBXSMM_SMM(REAL, UINT, M, N, K, A, B, C) { \\"
        print "    UINT libxsmm_i_, libxsmm_j_, libxsmm_k_; \\"
        print "    REAL *const libxsmm_c_ = (C); \\"
        print "    LIBXSMM_PRAGMA(vector nontemporal(libxsmm_c_)) \\"
        print "    LIBXSMM_PRAGMA(/*omp*/ simd collapse(2)) \\"
        print "    for (libxsmm_j_ = 0; libxsmm_j_ < (M); ++libxsmm_j_) { \\"
        print "      for (libxsmm_i_ = 0; libxsmm_i_ < (N); ++libxsmm_i_) { \\"
        print "        const UINT libxsmm_index_ = libxsmm_i_ * (M) + libxsmm_j_; \\"
        print "        REAL libxsmm_r_ = libxsmm_c_[libxsmm_index_]; \\"
        print "        LIBXSMM_PRAGMA(unroll(16)) \\"
        print "        LIBXSMM_PRAGMA(/*omp*/ simd reduction(+:libxsmm_r_)) \\"
        print "        for (libxsmm_k_ = 0; libxsmm_k_ < (K); ++libxsmm_k_) { \\"
        print "          libxsmm_r_ += (A)[libxsmm_k_*M+libxsmm_j_] * (B)[libxsmm_i_*K+libxsmm_k_]; \\"
        print "        } \\"
        print "        libxsmm_c_[libxsmm_index_] = libxsmm_r_; \\"
        print "      } \\"
        print "    } \\"
        print "  }"
    print "#endif"

def make_typeflag(Real):
    return ["s", "d"]["float" != Real]

def make_typepfix(Real):
    return ["", "d"]["float" != Real]

def create_mm(Real, RowMajor, M, N, K):
    if (0 != RowMajor):
        Rows, Cols = N, M
        l1, l2 = "b", "a"
    else:
        Rows, Cols = M, N
        l1, l2 = "a", "b"
    iparts = int(math.floor(Rows / 8))
    if (0 == (Rows % 8)):
        mnparts = iparts
    else:
        mnparts = iparts + 1
    print "LIBXSMM_EXTERN_C void libxsmm_" + make_typeflag(Real) + "mm_" + str(M) + "_" + str(N) + "_" + str(K) + "(const " + Real + "* a, const " + Real + "* b, " + Real + "* c)"
    print "{"
    print "#if defined(__MIC__)"
    print "  int i;"
    for mn in range(0, 8 * mnparts, 8):
        print "  {"
        mnm = min(mn + 7, Rows - 1)
        maskval = (1 << (mnm - mn + 1)) - 1
        print "    const __m512" + make_typepfix(Real) + " x" + l1 + "[] = {"
        for k in range(0, K):
            print "      MM512_MASK_LOADU_PD(" + l1 + " + " + str(Rows * k) + " + " + str(mn) + ", " + str(maskval) + "),"
        print "    };"
        print
        print "    for (i = 0; i < " + str(Cols) + "; ++i) {"
        print "      const int index = i * " + str(Rows) + " + " + str(mn) + ";"
        print "      __m512" + make_typepfix(Real) + " x" + l2 + "[" + str(K) + "], xc = MM512_MASK_LOADU_PD(c + index, " + str(maskval) + ");"
        for k in range(0, K):
            print "      x" + l2 + "[" + str(k) + "] = _mm512_set1_pd(" + l2 + "[i*" + str(K) + "+" + str(k) + "]);"
            print "      xc = _mm512_mask3_fmadd_pd(xa[" + str(k) + "], xb[" + str(k) + "], xc, " + str(maskval) + ");"
        print "      MM512_MASK_STOREU_PD(c + index, xc, " + str(maskval) + ");"
        print "    }"
        print "  }"
    print "#else"
    print "  LIBXSMM_SMM(" + Real + ", int, " + str(M) + ", " + str(N) + ", " + str(K) + ", a, b, c);"
    print "#endif"
    print "}"

def load_dims(dims):
    dims = list(map(int, dims)) #; dims.sort()
    return dims

if (6 <= len(sys.argv)):
    RowMajor = int(sys.argv[1])

    if (0 > int(sys.argv[2])):
        print "#include \"libxsmm.h\""
        print "#include <libxsmm_knc.h>"
        print
        print
        M = int(sys.argv[3])
        N = int(sys.argv[4])
        K = int(sys.argv[5])
        # Note: create_mm is not yet ready to generate the single-precision implementation
        print "LIBXSMM_EXTERN_C void libxsmm_smm_" + str(M) + "_" + str(N) + "_" + str(K) + "(const float* a, const float* b, float* c)"
        print "{"
        print "  LIBXSMM_SMM(float, int, " + str(M) + ", " + str(N) + ", " + str(K) + ", a, b, c);"
        print "}"
        print
        print
        create_mm("double", RowMajor, M, N, K)
    elif (7 <= len(sys.argv)):
        dimsM = load_dims(sys.argv[5:5+int(sys.argv[3])])
        dimsN = load_dims(sys.argv[5+int(sys.argv[3]):5+int(sys.argv[3])+int(sys.argv[4])])
        dimsK = load_dims(sys.argv[5+int(sys.argv[3])+int(sys.argv[4]):])
        maxMNK = int(sys.argv[2])
        for m in dimsM:
            for n in dimsN:
                for k in dimsK:
                    maxMNK = max(maxMNK, m * n * k)
        create_macros(RowMajor, maxMNK)
    else:
        sys.stderr.write(sys.argv[0] + ": wrong number of arguments!\n")
else:
    sys.stderr.write(sys.argv[0] + ": wrong number of arguments!\n")
<commit_msg>Distinct between masked and non-masked LOAD/STORE instructions; this is not necessary in terms of performance but perhaps for clarity. Using non-temporal LOAD/STORE instructions for the result matrix gives 5-10% better performance.<commit_after>###############################################################################
## Copyright (c) 2013-2014, Intel Corporation                                ##
## All rights reserved.                                                      ##
##                                                                           ##
## Redistribution and use in source and binary forms, with or without        ##
## modification, are permitted provided that the following conditions        ##
## are met:                                                                  ##
## 1. Redistributions of source code must retain the above copyright         ##
##    notice, this list of conditions and the following disclaimer.          ##
## 2. Redistributions in binary form must reproduce the above copyright      ##
##    notice, this list of conditions and the following disclaimer in the    ##
##    documentation and/or other materials provided with the distribution.   ##
## 3. Neither the name of the copyright holder nor the names of its          ##
##    contributors may be used to endorse or promote products derived        ##
##    from this software without specific prior written permission.          ##
##                                                                           ##
## THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS       ##
## "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT         ##
## LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR     ##
## A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT      ##
## HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,    ##
## SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED  ##
## TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR    ##
## PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF    ##
## LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING      ##
## NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS        ##
## SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.              ##
###############################################################################
## Christopher Dahnken (Intel Corp.), Hans Pabst (Intel Corp.),
## Alfio Lazzaro (CRAY Inc.), and Gilles Fourestey (CSCS)
###############################################################################
import math
import sys

def create_macros(RowMajor, maxMNK):
    print "#define LIBXSMM_MAX_MNK " + str(maxMNK)
    if (0 != RowMajor):
        print "#define LIBXSMM_ROW_MAJOR 1"
        print "#define LIBXSMM_COL_MAJOR 0"
        print
        print "#define LIBXSMM_BLASMM(REAL, UINT, M, N, K, A, B, C) { \\"
        print "  REAL libxsmm_alpha_ = 1, libxsmm_beta_ = 1; \\"
        print "  UINT libxsmm_m_ = (M), libxsmm_n_ = (N), libxsmm_k_ = (K); \\"
        print "  char libxsmm_trans_ = 'N'; \\"
        print "  LIBXSMM_BLASPREC(, REAL, gemm)(&libxsmm_trans_, &libxsmm_trans_, \\"
        print "    &libxsmm_n_, &libxsmm_m_, &libxsmm_k_, \\"
        print "    &libxsmm_alpha_, (REAL*)(B), &libxsmm_n_, (REAL*)(A), &libxsmm_k_, \\"
        print "    &libxsmm_beta_, (C), &libxsmm_n_); \\"
        print "}"
    else:
        print "#define LIBXSMM_ROW_MAJOR 0"
        print "#define LIBXSMM_COL_MAJOR 1"
        print
        print "#define LIBXSMM_BLASMM(REAL, UINT, M, N, K, A, B, C) { \\"
        print "  UINT libxsmm_m_ = (M), libxsmm_n_ = (N), libxsmm_k_ = (K); \\"
        print "  REAL libxsmm_alpha_ = 1, libxsmm_beta_ = 1; \\"
        print "  char libxsmm_trans_ = 'N'; \\"
        print "  LIBXSMM_BLASPREC(, REAL, gemm)(&libxsmm_trans_, &libxsmm_trans_, \\"
        print "    &libxsmm_m_, &libxsmm_n_, &libxsmm_k_, \\"
        print "    &libxsmm_alpha_, (REAL*)(A), &libxsmm_k_, (REAL*)(B), &libxsmm_n_, \\"
        print "    &libxsmm_beta_, (C), &libxsmm_n_); \\"
        print "}"
    print
    print "#if defined(MKL_DIRECT_CALL_SEQ) || defined(MKL_DIRECT_CALL)"
    print "# define LIBXSMM_SMM(REAL, UINT, M, N, K, A, B, C) LIBXSMM_BLASMM(REAL, UINT, M, N, K, A, B, C)"
    print "#else"
    if (0 != RowMajor):
        print "# define LIBXSMM_SMM(REAL, UINT, M, N, K, A, B, C) { \\"
        print "    UINT libxsmm_i_, libxsmm_j_, libxsmm_k_; \\"
        print "    REAL *const libxsmm_c_ = (C); \\"
        print "    LIBXSMM_PRAGMA(vector nontemporal(libxsmm_c_)) \\"
        print "    LIBXSMM_PRAGMA(/*omp*/ simd collapse(2)) \\"
        print "    for (libxsmm_j_ = 0; libxsmm_j_ < (N); ++libxsmm_j_) { \\"
        print "      for (libxsmm_i_ = 0; libxsmm_i_ < (M); ++libxsmm_i_) { \\"
        print "        const UINT libxsmm_index_ = libxsmm_i_ * (N) + libxsmm_j_; \\"
        print "        REAL libxsmm_r_ = libxsmm_c_[libxsmm_index_]; \\"
        print "        LIBXSMM_PRAGMA(unroll(16)) \\"
        print "        LIBXSMM_PRAGMA(/*omp*/ simd reduction(+:libxsmm_r_)) \\"
        print "        for (libxsmm_k_ = 0; libxsmm_k_ < (K); ++libxsmm_k_) { \\"
        print "          libxsmm_r_ += (A)[libxsmm_i_*K+libxsmm_k_] * (B)[libxsmm_k_*N+libxsmm_j_]; \\"
        print "        } \\"
        print "        libxsmm_c_[libxsmm_index_] = libxsmm_r_; \\"
        print "      } \\"
        print "    } \\"
        print "  }"
    else:
        print "# define LIBXSMM_SMM(REAL, UINT, M, N, K, A, B, C) { \\"
        print "    UINT libxsmm_i_, libxsmm_j_, libxsmm_k_; \\"
        print "    REAL *const libxsmm_c_ = (C); \\"
        print "    LIBXSMM_PRAGMA(vector nontemporal(libxsmm_c_)) \\"
        print "    LIBXSMM_PRAGMA(/*omp*/ simd collapse(2)) \\"
        print "    for (libxsmm_j_ = 0; libxsmm_j_ < (M); ++libxsmm_j_) { \\"
        print "      for (libxsmm_i_ = 0; libxsmm_i_ < (N); ++libxsmm_i_) { \\"
        print "        const UINT libxsmm_index_ = libxsmm_i_ * (M) + libxsmm_j_; \\"
        print "        REAL libxsmm_r_ = libxsmm_c_[libxsmm_index_]; \\"
        print "        LIBXSMM_PRAGMA(unroll(16)) \\"
        print "        LIBXSMM_PRAGMA(/*omp*/ simd reduction(+:libxsmm_r_)) \\"
        print "        for (libxsmm_k_ = 0; libxsmm_k_ < (K); ++libxsmm_k_) { \\"
        print "          libxsmm_r_ += (A)[libxsmm_k_*M+libxsmm_j_] * (B)[libxsmm_i_*K+libxsmm_k_]; \\"
        print "        } \\"
        print "        libxsmm_c_[libxsmm_index_] = libxsmm_r_; \\"
        print "      } \\"
        print "    } \\"
        print "  }"
    print "#endif"

def make_typeflag(Real):
    return ["s", "d"]["float" != Real]

def make_typepfix(Real):
    return ["", "d"]["float" != Real]

def create_mm(Real, M, N, K, RowMajor):
    if (0 != RowMajor):
        Rows, Cols = N, M
        l1, l2 = "b", "a"
    else:
        Rows, Cols = M, N
        l1, l2 = "a", "b"
    iparts = int(math.floor(Rows / 8))
    if (0 == (Rows % 8)):
        mnparts = iparts
    else:
        mnparts = iparts + 1
    print "LIBXSMM_EXTERN_C void libxsmm_" + make_typeflag(Real) + "mm_" + str(M) + "_" + str(N) + "_" + str(K) + "(const " + Real + "* a, const " + Real + "* b, " + Real + "* c)"
    print "{"
    print "#if defined(__MIC__)"
    print "  int i;"
    for mn in range(0, 8 * mnparts, 8):
        print "  {"
        mnm = min(mn + 7, Rows - 1)
        maskval = (1 << (mnm - mn + 1)) - 1
        print "    const __m512" + make_typepfix(Real) + " x" + l1 + "[] = {"
        if (255 != maskval):
            mask_inst, mask_argv, mask3 = "MASK_", ", " + str(maskval), "mask3_"
        else:
            mask_inst, mask_argv, mask3 = "", "", ""
        for k in range(0, K):
            print "      MM512_" + mask_inst + "LOADU_PD(" + l1 + " + " + str(Rows * k) + " + " + str(mn) + mask_argv + "),"
        print "    };"
        print
        print "    for (i = 0; i < " + str(Cols) + "; ++i) {"
        print "      const int index = i * " + str(Rows) + " + " + str(mn) + ";"
        print "      __m512" + make_typepfix(Real) + " xc = MM512_" + mask_inst + "LOADNTU_PD(c + index" + mask_argv + "), x" + l2 + "[" + str(K) + "];"
        for k in range(0, K):
            print "      x" + l2 + "[" + str(k) + "] = _mm512_set1_pd(" + l2 + "[i*" + str(K) + "+" + str(k) + "]);"
            print "      xc = _mm512_" + mask3 + "fmadd_pd(xa[" + str(k) + "], xb[" + str(k) + "], xc" + mask_argv + ");"
        print "      MM512_" + mask_inst + "STORENTU_PD(c + index, xc" + mask_argv + ");"
        print "    }"
        print "  }"
    print "#else"
    print "  LIBXSMM_SMM(" + Real + ", int, " + str(M) + ", " + str(N) + ", " + str(K) + ", a, b, c);"
    print "#endif"
    print "}"

def load_dims(dims):
    dims = list(map(int, dims)) #; dims.sort()
    return dims

if (6 <= len(sys.argv)):
    RowMajor = int(sys.argv[1])

    if (0 > int(sys.argv[2])):
        print "#include <libxsmm_knc.h>"
        print
        print
        M = int(sys.argv[3])
        N = int(sys.argv[4])
        K = int(sys.argv[5])
        # Note: create_mm is not yet ready to generate the single-precision implementation
        print "LIBXSMM_EXTERN_C void libxsmm_smm_" + str(M) + "_" + str(N) + "_" + str(K) + "(const float* a, const float* b, float* c)"
        print "{"
        print "  LIBXSMM_SMM(float, int, " + str(M) + ", " + str(N) + ", " + str(K) + ", a, b, c);"
        print "}"
        print
        print
        create_mm("double", M, N, K, RowMajor)
    elif (7 <= len(sys.argv)):
        dimsM = load_dims(sys.argv[5:5+int(sys.argv[3])])
        dimsN = load_dims(sys.argv[5+int(sys.argv[3]):5+int(sys.argv[3])+int(sys.argv[4])])
        dimsK = load_dims(sys.argv[5+int(sys.argv[3])+int(sys.argv[4]):])
        maxMNK = int(sys.argv[2])
        for m in dimsM:
            for n in dimsN:
                for k in dimsK:
                    maxMNK = max(maxMNK, m * n * k)
        create_macros(RowMajor, maxMNK)
    else:
        sys.stderr.write(sys.argv[0] + ": wrong number of arguments!\n")
else:
    sys.stderr.write(sys.argv[0] + ": wrong number of arguments!\n")
<|endoftext|>


<commit_before>__version__ = "1.6.3"
__package_name__ = "panoply-python-sdk"
<commit_msg>Bump version<commit_after>__version__ = "2.0.0"
__package_name__ = "panoply-python-sdk"
<|endoftext|>