<commit_before>
# ovirt-hosted-engine-setup -- ovirt hosted engine setup
# Copyright (C) 2017 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

"""Ansible Utils"""

import gettext
import json
import os
import random
import string
import subprocess
import tempfile
import time

from otopi import base

from ovirt_hosted_engine_setup import constants as ohostedcons

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-hosted-engine-setup')

# TODO do this nicely
_FILTERED_VARS = (
    'ADMIN_PASSWORD',
    'APPLIANCE_PASSWORD',
    'ISCSI_PASSWORD',
    'ISCSI_DISCOVER_PASSWORD',
    'ROOTPWD',
    'he_appliance_password',
    'he_admin_password',
    'he_iscsi_password',
    'he_iscsi_discover_password',
    'ansible_ssh_pass',
)

_FILTERED_REs = (
    'BEGIN PRIVATE KEY(?P<filter>.*)END PRIVATE KEY',
)

_EXTRA_VARS_FOR_FILTERING = {
    'he_filtered_tokens_vars': list(_FILTERED_VARS),
    'he_filtered_tokens_re': list(_FILTERED_REs),
}

class AnsibleHelper(base.Base):

    def __init__(
        self,
        playbook_name=ohostedcons.FileLocations.HE_AP_TRIGGER_ROLE,
        custom_path=None,
        extra_vars={},
        user_extra_vars=None,
        inventory_source='localhost,',
        raise_on_error=True,
        tags=None,
        skip_tags='always',
    ):
        super(AnsibleHelper, self).__init__()
        self._playbook_name = playbook_name
        self._module_path = custom_path if custom_path \
            else ohostedcons.FileLocations.HOSTED_ENGINE_ANSIBLE_PATH
        self._playbook_path = os.path.join(
            self._module_path,
            self._playbook_name
        )
        self._inventory_source = inventory_source
        self._extra_vars = extra_vars
        self._extra_vars.update(_EXTRA_VARS_FOR_FILTERING)
        self._user_extra_vars = user_extra_vars
        self._cb_results = {}
        self._raise_on_error = raise_on_error
        self._tags = tags
        self._skip_tags = skip_tags

    def _process_output(self, d):
        try:
            data = json.loads(d)
            if (
                ohostedcons.AnsibleCallback.TYPE in data and
                ohostedcons.AnsibleCallback.BODY in data
            ):
                t = data[ohostedcons.AnsibleCallback.TYPE]
                b = data[ohostedcons.AnsibleCallback.BODY]
                if t == ohostedcons.AnsibleCallback.DEBUG:
                    self.logger.debug(b)
                elif t == ohostedcons.AnsibleCallback.WARNING:
                    self.logger.warning(b)
                elif t == ohostedcons.AnsibleCallback.ERROR:
                    self.logger.error(b)
                elif t == ohostedcons.AnsibleCallback.INFO:
                    self.logger.info(b)
                elif t == ohostedcons.AnsibleCallback.RESULT:
                    self._cb_results = b
                else:
                    self.logger.error(_('Unknown data type: {t}').format(t=t))
        except Exception as e:
            self.logger.error(
                _('Failed decoding json data: {e} - "{d}"').format(
                    e=str(e),
                    d=d,
                )
            )

    def _format_tags_option(self, tags, tag_option):
        if tags and tag_option:
            if isinstance(tags, list) or isinstance(tags, tuple):
                tags = ','.join(tags)
            return '%s=%s' % (tag_option, tags)

        return ''

    def run(self):
        out_fd, out_path = tempfile.mkstemp()
        vars_fd, vars_path = tempfile.mkstemp()
        ansible_playbook_cmd = [
            '/bin/ansible-playbook',
            '--module-path={mp}'.format(mp=self._module_path),
            '--inventory={i}'.format(i=self._inventory_source),
            '--extra-vars=@{vf}'.format(vf=vars_path),
        ]
        if self._user_extra_vars:
            ansible_playbook_cmd.append(
                '--extra-vars={}'.format(self._user_extra_vars)
            )

        tags = self._format_tags_option(self._tags, '--tags')
        skip_tags = self._format_tags_option(self._skip_tags, '--skip-tags')
        if tags:
            ansible_playbook_cmd.append(tags)
        if skip_tags:
            ansible_playbook_cmd.append(skip_tags)

        ansible_playbook_cmd.append(self._playbook_path)

        env = os.environ.copy()
        env[ohostedcons.AnsibleCallback.OTOPI_CALLBACK_OF] = out_path
        env[
            'ANSIBLE_CALLBACK_WHITELIST'
        ] = '{com},{log}'.format(
            com=ohostedcons.AnsibleCallback.CALLBACK_NAME,
            log=ohostedcons.AnsibleCallback.LOGGER_CALLBACK_NAME,
        )
        env[
            'ANSIBLE_STDOUT_CALLBACK'
        ] = ohostedcons.AnsibleCallback.CALLBACK_NAME

        dname = os.path.splitext(self._playbook_name)[0]
        if self._tags:
            if isinstance(self._tags, list) or isinstance(self._tags, tuple):
                tag_name = self._tags[0]
            else:
                tag_name = self._tags
            dname = tag_name

        env[
            'HE_ANSIBLE_LOG_PATH'
        ] = os.path.join(
            ohostedcons.FileLocations.OVIRT_HOSTED_ENGINE_SETUP_LOGDIR,
            "%s-ansible-%s-%s-%s.log" % (
                ohostedcons.FileLocations.OVIRT_HOSTED_ENGINE_SETUP,
                dname,
                time.strftime("%Y%m%d%H%M%S"),
                ''.join(
                    [
                        random.choice(
                            string.ascii_lowercase +
                            string.digits
                        ) for i in range(6)
                    ]
                )
            )
        )

        self.logger.debug('ansible-playbook: cmd: %s' % ansible_playbook_cmd)
        self.logger.debug('ansible-playbook: out_path: %s' % out_path)
        self.logger.debug('ansible-playbook: vars_path: %s' % vars_path)
        self.logger.debug('ansible-playbook: env: %s' % env)

        rc = None
        with open(vars_path, 'w') as vars_fh:
            json.dump(self._extra_vars, vars_fh)
        with open(out_path, 'r') as out_fh:
            buffer = ''
            proc = subprocess.Popen(
                ansible_playbook_cmd,
                env=env,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            while True:
                output = out_fh.readline()
                time.sleep(0.1)
                if output == '' and proc.poll() is not None:
                    break
                if output:
                    buffer += output
                    if buffer[-1] == '\n':
                        self._process_output(buffer)
                        buffer = ''
            rc = proc.poll()
            self._cb_results['ansible-playbook_rc'] = rc
            self.logger.debug('ansible-playbook rc: {rc}'.format(rc=rc))
            while True:
                output = out_fh.readline()
                if output == '':
                    break
                if output:
                    self._process_output(output)
            self.logger.debug('ansible-playbook stdout:')
            for ln in proc.stdout:
                self.logger.debug(ln)
            self.logger.debug('ansible-playbook stderr:')
            for ln in proc.stderr:
                self.logger.error(ln)
            if rc != 0 and self._raise_on_error:
                raise RuntimeError(_('Failed executing ansible-playbook'))
        os.unlink(out_path)
        os.unlink(vars_path)
        return self._cb_results

# vim: expandtab tabstop=4 shiftwidth=4
<commit_msg>Filter out he_hashed_appliance_password<commit_after>
# ovirt-hosted-engine-setup -- ovirt hosted engine setup
# Copyright (C) 2017 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

"""Ansible Utils"""

import gettext
import json
import os
import random
import string
import subprocess
import tempfile
import time

from otopi import base

from ovirt_hosted_engine_setup import constants as ohostedcons

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-hosted-engine-setup')

# TODO do this nicely
_FILTERED_VARS = (
    'ADMIN_PASSWORD',
    'APPLIANCE_PASSWORD',
    'ISCSI_PASSWORD',
    'ISCSI_DISCOVER_PASSWORD',
    'ROOTPWD',
    'he_appliance_password',
    'he_hashed_appliance_password',
    'he_admin_password',
    'he_iscsi_password',
    'he_iscsi_discover_password',
    'ansible_ssh_pass',
)

_FILTERED_REs = (
    'BEGIN PRIVATE KEY(?P<filter>.*)END PRIVATE KEY',
)

_EXTRA_VARS_FOR_FILTERING = {
    'he_filtered_tokens_vars': list(_FILTERED_VARS),
    'he_filtered_tokens_re': list(_FILTERED_REs),
}

class AnsibleHelper(base.Base):

    def __init__(
        self,
        playbook_name=ohostedcons.FileLocations.HE_AP_TRIGGER_ROLE,
        custom_path=None,
        extra_vars={},
        user_extra_vars=None,
        inventory_source='localhost,',
        raise_on_error=True,
        tags=None,
        skip_tags='always',
    ):
        super(AnsibleHelper, self).__init__()
        self._playbook_name = playbook_name
        self._module_path = custom_path if custom_path \
            else ohostedcons.FileLocations.HOSTED_ENGINE_ANSIBLE_PATH
        self._playbook_path = os.path.join(
            self._module_path,
            self._playbook_name
        )
        self._inventory_source = inventory_source
        self._extra_vars = extra_vars
        self._extra_vars.update(_EXTRA_VARS_FOR_FILTERING)
        self._user_extra_vars = user_extra_vars
        self._cb_results = {}
        self._raise_on_error = raise_on_error
        self._tags = tags
        self._skip_tags = skip_tags

    def _process_output(self, d):
        try:
            data = json.loads(d)
            if (
                ohostedcons.AnsibleCallback.TYPE in data and
                ohostedcons.AnsibleCallback.BODY in data
            ):
                t = data[ohostedcons.AnsibleCallback.TYPE]
                b = data[ohostedcons.AnsibleCallback.BODY]
                if t == ohostedcons.AnsibleCallback.DEBUG:
                    self.logger.debug(b)
                elif t == ohostedcons.AnsibleCallback.WARNING:
                    self.logger.warning(b)
                elif t == ohostedcons.AnsibleCallback.ERROR:
                    self.logger.error(b)
                elif t == ohostedcons.AnsibleCallback.INFO:
                    self.logger.info(b)
                elif t == ohostedcons.AnsibleCallback.RESULT:
                    self._cb_results = b
                else:
                    self.logger.error(_('Unknown data type: {t}').format(t=t))
        except Exception as e:
            self.logger.error(
                _('Failed decoding json data: {e} - "{d}"').format(
                    e=str(e),
                    d=d,
                )
            )

    def _format_tags_option(self, tags, tag_option):
        if tags and tag_option:
            if isinstance(tags, list) or isinstance(tags, tuple):
                tags = ','.join(tags)
            return '%s=%s' % (tag_option, tags)

        return ''

    def run(self):
        out_fd, out_path = tempfile.mkstemp()
        vars_fd, vars_path = tempfile.mkstemp()
        ansible_playbook_cmd = [
            '/bin/ansible-playbook',
            '--module-path={mp}'.format(mp=self._module_path),
            '--inventory={i}'.format(i=self._inventory_source),
            '--extra-vars=@{vf}'.format(vf=vars_path),
        ]
        if self._user_extra_vars:
            ansible_playbook_cmd.append(
                '--extra-vars={}'.format(self._user_extra_vars)
            )

        tags = self._format_tags_option(self._tags, '--tags')
        skip_tags = self._format_tags_option(self._skip_tags, '--skip-tags')
        if tags:
            ansible_playbook_cmd.append(tags)
        if skip_tags:
            ansible_playbook_cmd.append(skip_tags)

        ansible_playbook_cmd.append(self._playbook_path)

        env = os.environ.copy()
        env[ohostedcons.AnsibleCallback.OTOPI_CALLBACK_OF] = out_path
        env[
            'ANSIBLE_CALLBACK_WHITELIST'
        ] = '{com},{log}'.format(
            com=ohostedcons.AnsibleCallback.CALLBACK_NAME,
            log=ohostedcons.AnsibleCallback.LOGGER_CALLBACK_NAME,
        )
        env[
            'ANSIBLE_STDOUT_CALLBACK'
        ] = ohostedcons.AnsibleCallback.CALLBACK_NAME

        dname = os.path.splitext(self._playbook_name)[0]
        if self._tags:
            if isinstance(self._tags, list) or isinstance(self._tags, tuple):
                tag_name = self._tags[0]
            else:
                tag_name = self._tags
            dname = tag_name

        env[
            'HE_ANSIBLE_LOG_PATH'
        ] = os.path.join(
            ohostedcons.FileLocations.OVIRT_HOSTED_ENGINE_SETUP_LOGDIR,
            "%s-ansible-%s-%s-%s.log" % (
                ohostedcons.FileLocations.OVIRT_HOSTED_ENGINE_SETUP,
                dname,
                time.strftime("%Y%m%d%H%M%S"),
                ''.join(
                    [
                        random.choice(
                            string.ascii_lowercase +
                            string.digits
                        ) for i in range(6)
                    ]
                )
            )
        )

        self.logger.debug('ansible-playbook: cmd: %s' % ansible_playbook_cmd)
        self.logger.debug('ansible-playbook: out_path: %s' % out_path)
        self.logger.debug('ansible-playbook: vars_path: %s' % vars_path)
        self.logger.debug('ansible-playbook: env: %s' % env)

        rc = None
        with open(vars_path, 'w') as vars_fh:
            json.dump(self._extra_vars, vars_fh)
        with open(out_path, 'r') as out_fh:
            buffer = ''
            proc = subprocess.Popen(
                ansible_playbook_cmd,
                env=env,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
            )
            while True:
                output = out_fh.readline()
                time.sleep(0.1)
                if output == '' and proc.poll() is not None:
                    break
                if output:
                    buffer += output
                    if buffer[-1] == '\n':
                        self._process_output(buffer)
                        buffer = ''
            rc = proc.poll()
            self._cb_results['ansible-playbook_rc'] = rc
            self.logger.debug('ansible-playbook rc: {rc}'.format(rc=rc))
            while True:
                output = out_fh.readline()
                if output == '':
                    break
                if output:
                    self._process_output(output)
            self.logger.debug('ansible-playbook stdout:')
            for ln in proc.stdout:
                self.logger.debug(ln)
            self.logger.debug('ansible-playbook stderr:')
            for ln in proc.stderr:
                self.logger.error(ln)
            if rc != 0 and self._raise_on_error:
                raise RuntimeError(_('Failed executing ansible-playbook'))
        os.unlink(out_path)
        os.unlink(vars_path)
        return self._cb_results

# vim: expandtab tabstop=4 shiftwidth=4
<|endoftext|>


<commit_before>import copy
import itertools
import json

from .archive import SGFLocator, find_sgfs, tarball_iterator
from ..dataloader.goboard import GoBoard
from ..gosgf import Sgf_game

__all__ = [
    'CorpusIndex',
    'build_index',
    'load_index',
    'store_index',
]

def _sequence(game_record):
    """Extract game moves from a game record.

    The main sequence includes lots of stuff that is not actual game
    moves.
    """
    seq = []
    for item in game_record.get_main_sequence():
        color, move = item.get_move()
        # color == None is entries that are not actual game play
        # move == None is a pass, which in theory we could try to
        # predict, but not yet
        if color is not None and move is not None:
            seq.append((color, move))
    return seq

class CorpusIndex(object):
    def __init__(self, physical_files, chunk_size, boundaries):
        self.physical_files = list(sorted(physical_files))
        self.chunk_size = chunk_size
        self.boundaries = list(boundaries)

    @property
    def num_chunks(self):
        return len(self.boundaries)

    def serialize(self):
        return {
            'physical_files': self.physical_files,
            'chunk_size': self.chunk_size,
            'boundaries': [boundary.serialize() for boundary in self.boundaries],
        }

    @classmethod
    def deserialize(cls, serialized):
        return cls(
            serialized['physical_files'],
            serialized['chunk_size'],
            [Pointer.deserialize(raw_boundary) for raw_boundary in serialized['boundaries']])

    def get_chunk(self, chunk_number):
        assert 0 <= chunk_number < self.num_chunks
        chunk_start = self.boundaries[chunk_number]
        iterator = iter(self._generate_examples(chunk_start.locator))
        # Skip to the appropriate move in the current game.
        for _ in range(chunk_start.position):
            next(iterator)
        return itertools.islice(iterator, self.chunk_size)

    def _generate_examples(self, start):
        """
        Args:
            start (SGFLocator)
        """
        start_file_idx = self.physical_files.index(start.physical_file)
        for physical_file in self.physical_files[start_file_idx:]:
            for sgf in self._generate_games(physical_file):
                if sgf.locator < start:
                    continue
                game_record = Sgf_game.from_string(sgf.contents)
                board = GoBoard(19)
                # Set up the handicap.
                if game_record.get_handicap() > 0:
                    for setup in game_record.get_root().get_setup_stones():
                        for move in setup:
                            board.apply_move('b', move)
                for i, (color, move) in enumerate(_sequence(game_record)):
                    yield copy.deepcopy(board), color, move
                    if move is not None:
                        board.apply_move(color, move)

    def _generate_games(self, physical_file):
        with tarball_iterator(physical_file) as tarball:
            for sgf in tarball:
                yield sgf

class Pointer(object):
    """Identifies a specific training example inside a corpus."""
    def __init__(self, locator, position):
        self.locator = locator
        self.position = position

    def __str__(self):
        return '%s:%d' % (self.locator, self.position)

    def serialize(self):
        return {
            'locator': self.locator.serialize(),
            'position': self.position,
        }

    @classmethod
    def deserialize(cls, serialized):
        return cls(
            SGFLocator.deserialize(serialized['locator']),
            serialized['position']
        )

def build_index(path, chunk_size):
    """Index all SGF files found in the given location.

    This will include SGF that are contained inside zip or tar archives.
    """
    physical_files = set()
    boundaries = []
    examples_needed = 0
    for sgf in find_sgfs(path):
        physical_files.add(sgf.locator.physical_file)
        if examples_needed == 0:
            # The start of this SGF is a chunk boundary.
            boundaries.append(Pointer(sgf.locator, 0))
            examples_needed = chunk_size
        game_record = Sgf_game.from_string(sgf.contents)
        num_positions = len(_sequence(game_record))
        if examples_needed < num_positions:
            # The start of the next chunk is inside this SGF.
            boundaries.append(Pointer(sgf.locator, examples_needed))
            remaining_examples = num_positions - examples_needed
            examples_needed = chunk_size - remaining_examples
        else:
            # This SGF is entirely contained within the current chunk.
            examples_needed -= num_positions

    return CorpusIndex(physical_files, chunk_size, boundaries)

def load_index(input_stream):
    return CorpusIndex.deserialize(json.load(input_stream))

def store_index(index, output_stream):
    json.dump(index.serialize(), output_stream)
<commit_msg>Handle exceptions in single SGFs.<commit_after>import copy
import itertools
import json

from .archive import SGFLocator, find_sgfs, tarball_iterator
from ..dataloader.goboard import GoBoard
from ..gosgf import Sgf_game

__all__ = [
    'CorpusIndex',
    'build_index',
    'load_index',
    'store_index',
]

def _sequence(game_record):
    """Extract game moves from a game record.

    The main sequence includes lots of stuff that is not actual game
    moves.
    """
    seq = []
    for item in game_record.get_main_sequence():
        color, move = item.get_move()
        # color == None is entries that are not actual game play
        # move == None is a pass, which in theory we could try to
        # predict, but not yet
        if color is not None and move is not None:
            seq.append((color, move))
    return seq

class CorpusIndex(object):
    def __init__(self, physical_files, chunk_size, boundaries):
        self.physical_files = list(sorted(physical_files))
        self.chunk_size = chunk_size
        self.boundaries = list(boundaries)

    @property
    def num_chunks(self):
        return len(self.boundaries)

    def serialize(self):
        return {
            'physical_files': self.physical_files,
            'chunk_size': self.chunk_size,
            'boundaries': [boundary.serialize() for boundary in self.boundaries],
        }

    @classmethod
    def deserialize(cls, serialized):
        return cls(
            serialized['physical_files'],
            serialized['chunk_size'],
            [Pointer.deserialize(raw_boundary) for raw_boundary in serialized['boundaries']])

    def get_chunk(self, chunk_number):
        assert 0 <= chunk_number < self.num_chunks
        chunk_start = self.boundaries[chunk_number]
        iterator = iter(self._generate_examples(chunk_start.locator))
        # Skip to the appropriate move in the current game.
        for _ in range(chunk_start.position):
            next(iterator)
        return itertools.islice(iterator, self.chunk_size)

    def _generate_examples(self, start):
        """
        Args:
            start (SGFLocator)
        """
        start_file_idx = self.physical_files.index(start.physical_file)
        for physical_file in self.physical_files[start_file_idx:]:
            for sgf in self._generate_games(physical_file):
                if sgf.locator < start:
                    continue
                board = GoBoard(19)
                try:
                    game_record = Sgf_game.from_string(sgf.contents)
                    # Set up the handicap.
                    if game_record.get_handicap() > 0:
                        for setup in game_record.get_root().get_setup_stones():
                            for move in setup:
                                board.apply_move('b', move)
                    for i, (color, move) in enumerate(_sequence(game_record)):
                        yield copy.deepcopy(board), color, move
                        if move is not None:
                            board.apply_move(color, move)
                except ValueError:
                    print("Invalid SGF data, skipping game record %s" % (sgf,))
                    print("Board was:\n%s" % (board,))

    def _generate_games(self, physical_file):
        with tarball_iterator(physical_file) as tarball:
            for sgf in tarball:
                yield sgf

class Pointer(object):
    """Identifies a specific training example inside a corpus."""
    def __init__(self, locator, position):
        self.locator = locator
        self.position = position

    def __str__(self):
        return '%s:%d' % (self.locator, self.position)

    def serialize(self):
        return {
            'locator': self.locator.serialize(),
            'position': self.position,
        }

    @classmethod
    def deserialize(cls, serialized):
        return cls(
            SGFLocator.deserialize(serialized['locator']),
            serialized['position']
        )

def build_index(path, chunk_size):
    """Index all SGF files found in the given location.

    This will include SGF that are contained inside zip or tar archives.
    """
    physical_files = set()
    boundaries = []
    examples_needed = 0
    for sgf in find_sgfs(path):
        physical_files.add(sgf.locator.physical_file)
        if examples_needed == 0:
            # The start of this SGF is a chunk boundary.
            boundaries.append(Pointer(sgf.locator, 0))
            examples_needed = chunk_size
        game_record = Sgf_game.from_string(sgf.contents)
        num_positions = len(_sequence(game_record))
        if examples_needed < num_positions:
            # The start of the next chunk is inside this SGF.
            boundaries.append(Pointer(sgf.locator, examples_needed))
            remaining_examples = num_positions - examples_needed
            examples_needed = chunk_size - remaining_examples
        else:
            # This SGF is entirely contained within the current chunk.
            examples_needed -= num_positions

    return CorpusIndex(physical_files, chunk_size, boundaries)

def load_index(input_stream):
    return CorpusIndex.deserialize(json.load(input_stream))

def store_index(index, output_stream):
    json.dump(index.serialize(), output_stream)
<|endoftext|>


<commit_before>"""Work out which LIF files need to be unpacked, and unpack them."""

import os
import errno
import argparse

from pprint import pprint

from jicbioimage.core.io import DataManager, FileBackend

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST:
            pass
        else: raise

def get_data_manager(output_directory):
    """Return FileBackend instance."""

    backend_dir = os.path.join(output_directory, '.backend')
    if not os.path.isdir(backend_dir):
        os.mkdir(backend_dir)
    backend = FileBackend(backend_dir)
    return DataManager(backend)

def is_directory_processed(output_directory):

    return False

def unpack_series_in_single_file(filename, output_directory):
    """Unpack the series contained in a particular image file. Returns

    backend_dir_path, (image_filename, series_identifier, output_path)

    where:

    image_filename is the name of the image file passed in to the function
    series_identifier is the numerical identifier of the series, from 0 onwards
    output_path is target directory for analysed data from the series.
    """

    data_manager = get_data_manager(output_directory)
    microscopy_collection = data_manager.load(filename)

    file_basename = os.path.basename(filename)
    file_output_directory = os.path.join(output_directory, file_basename)

    mkdir_p(file_output_directory)

    process_list = []
    series_ids = microscopy_collection.series
    for sid in series_ids:
        series_name = "series_{}".format(sid)
        series_output_directory = os.path.join(file_output_directory, series_name)
        mkdir_p(series_output_directory)

        if not is_directory_processed(series_output_directory):
            process_list.append((filename, sid, series_output_directory))

    return output_directory, process_list

def unpack_all_series_in_directory(input_directory, output_directory):
    """Given an input directory, find all of the image files in that directory,
    unpack them, and return a list of files and series to process, together
    with output directories into which processed output should be written."""

    all_input_files = os.listdir(input_directory)

    process_list = []

    for input_filename in all_input_files:
        fq_input_filename = os.path.join(input_directory, input_filename)
        process_commands = unpack_series_in_single_file(fq_input_filename,
                                                        output_directory)
        process_list += process_commands

    return output_directory, process_list

def main():

    parser = argparse.ArgumentParser(__doc__)
    parser.add_argument('input_directory', help='Input directory containing LIF files to unpack')
    parser.add_argument('output_directory', help='Directory into which output should be written')

    args = parser.parse_args()

    process_list = unpack_all_series_in_directory(args.input_directory, args.output_directory)

    pprint(process_list)

if __name__ == '__main__':
    main()
<commit_msg>Fixed defect in unpack_all_series_in_directory<commit_after>"""Work out which LIF files need to be unpacked, and unpack them."""

import os
import errno
import argparse

from pprint import pprint

from jicbioimage.core.io import DataManager, FileBackend

def mkdir_p(path):
    try:
        os.makedirs(path)
    except OSError as exc:
        if exc.errno == errno.EEXIST:
            pass
        else: raise

def get_data_manager(output_directory):
    """Return FileBackend instance."""

    backend_dir = os.path.join(output_directory, '.backend')
    if not os.path.isdir(backend_dir):
        os.mkdir(backend_dir)
    backend = FileBackend(backend_dir)
    return DataManager(backend)

def is_directory_processed(output_directory):

    return False

def unpack_series_in_single_file(filename, output_directory):
    """Unpack the series contained in a particular image file. Returns

    backend_dir_path, (image_filename, series_identifier, output_path)

    where:

    image_filename is the name of the image file passed in to the function
    series_identifier is the numerical identifier of the series, from 0 onwards
    output_path is target directory for analysed data from the series.
    """

    data_manager = get_data_manager(output_directory)
    microscopy_collection = data_manager.load(filename)

    file_basename = os.path.basename(filename)
    file_output_directory = os.path.join(output_directory, file_basename)

    mkdir_p(file_output_directory)

    process_list = []
    series_ids = microscopy_collection.series
    for sid in series_ids:
        series_name = "series_{}".format(sid)
        series_output_directory = os.path.join(file_output_directory, series_name)
        mkdir_p(series_output_directory)

        if not is_directory_processed(series_output_directory):
            process_list.append((filename, sid, series_output_directory))

    return output_directory, process_list

def unpack_all_series_in_directory(input_directory, output_directory):
    """Given an input directory, find all of the image files in that directory,
    unpack them, and return a list of files and series to process, together
    with output directories into which processed output should be written."""

    all_input_files = os.listdir(input_directory)

    process_list = []

    for input_filename in all_input_files:
        fq_input_filename = os.path.join(input_directory, input_filename)
        _, process_commands = unpack_series_in_single_file(fq_input_filename,
                                                        output_directory)
        process_list += process_commands

    return output_directory, process_list

def main():

    parser = argparse.ArgumentParser(__doc__)
    parser.add_argument('input_directory', help='Input directory containing LIF files to unpack')
    parser.add_argument('output_directory', help='Directory into which output should be written')

    args = parser.parse_args()

    process_list = unpack_all_series_in_directory(args.input_directory, args.output_directory)

    pprint(process_list)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import hackarena.constants
from hackarena.game_objects import BaseGameObject

class Team(BaseGameObject):

    def __init__(self, color):
        self.color = color
        self.kills = 0
        self.building_max_hp = 600
        self.building_hp = 600
        self.building_position = {
            'x': 32 if color == 'blue' else hackarena.constants.MAP_TILES_WIDTH * 16 - 96,
            'y': 32 if color == 'blue' else hackarena.constants.MAP_TILES_HEIGHT * 16 - 96,
        }
        self.building_size = {
            'width': 48,
            'height': 80,
        }
        self.players = []

    def add_player(self, player):
        self.players.append(player)

    def remove_player(self, player):
        self.players.remove(player)
<commit_msg>Moved the towers a bit for better collisions<commit_after># -*- coding: utf-8 -*-
import hackarena.constants
from hackarena.game_objects import BaseGameObject

class Team(BaseGameObject):

    def __init__(self, color):
        self.color = color
        self.kills = 0
        self.building_max_hp = 600
        self.building_hp = 600
        self.building_position = {
            'x': 40 if color == 'blue' else hackarena.constants.MAP_TILES_WIDTH * 16 - 88,
            'y': 32 if color == 'blue' else hackarena.constants.MAP_TILES_HEIGHT * 16 - 96,
        }
        self.building_size = {
            'width': 48,
            'height': 80,
        }
        self.players = []

    def add_player(self, player):
        self.players.append(player)

    def remove_player(self, player):
        self.players.remove(player)
<|endoftext|>


<commit_before>class Message(object):
  '''
  Represents an AMQP message.
  '''

  def __init__(self, body=None, delivery_info=None, **properties):
    if isinstance(body, unicode):
      if properties.get('content_encoding', None) is None:
        properties['content_encoding'] = 'utf-8'
      body = body.encode(properties['content_encoding'])

    self._body = body
    self._delivery_info = delivery_info
    self._properties = properties

  @property
  def body(self):
    return self._body

  def __len__(self):
    return len( self._body )

  def __nonzero__(self):
    '''Have to define this because length is defined.'''
    return True

  def __eq__(self, other):
    if isinstance(other,Message):
      return self._properties == other._properties and \
             self._body == other._body
    return False

  @property
  def delivery_info(self):
    return self._delivery_info

  @property
  def properties(self):
    return self._properties

  def __str__(self):
    return "Message[body: %s, delivery_info: %s, properties: %s]"%\
      ( str(self._body).encode('string_escape'), self._delivery_info, self._properties )
<commit_msg>Trivial change to Message.<commit_after>class Message(object):
  '''
  Represents an AMQP message.
  '''

  def __init__(self, body=None, delivery_info=None, **properties):
    if isinstance(body, unicode):
      if 'content_encoding' not in properties:
        properties['content_encoding'] = 'utf-8'
      body = body.encode(properties['content_encoding'])

    self._body = body
    self._delivery_info = delivery_info
    self._properties = properties

  @property
  def body(self):
    return self._body

  def __len__(self):
    return len( self._body )

  def __nonzero__(self):
    '''Have to define this because length is defined.'''
    return True

  def __eq__(self, other):
    if isinstance(other,Message):
      return self._properties == other._properties and \
             self._body == other._body
    return False

  @property
  def delivery_info(self):
    return self._delivery_info

  @property
  def properties(self):
    return self._properties

  def __str__(self):
    return "Message[body: %s, delivery_info: %s, properties: %s]"%\
      ( str(self._body).encode('string_escape'), self._delivery_info, self._properties )
<|endoftext|>


<commit_before>#builds the model from existing data
import inspect,os
import markovify
from textstat.textstat import textstat

#Read files from directory
#Add their contents to a single string
#Make a model from this string
#generate first sentence from this model

#
#"C:\CS Stuff\HaikuGen\\tests\Wordsworth.txt"
class train():
    dir=os.path.dirname(os.path.dirname(os.path.abspath(inspect.stack()[0][1])))

    def __init__(self,line_limits=[5,7,5],file1=os.path.join(dir,"tests\Macbeth-Shakespeare.txt"),file2=os.path.join(dir,"tests\Wordsworth.txt")):
        self.line_limits=line_limits
        self.file1=file1
        self.file2=file2

    def main(self):
        f1 = open(self.file1)
        text = f1.read()

        f2 = open(self.file2)
        text += f2.read()

        textModel = markovify.NewlineText(text)
        haiku=""
        for i in range(0,3):
            while True:
                sent=textModel.make_sentence()
                if(not sent==None) and (textstat.syllable_count(sent)<self.line_limits[i]):
                    haiku+=sent+"\n"
                    break

        print(haiku)

if __name__=="__main__":
    train.main(train())<commit_msg>Code cleaned up.<commit_after>#builds the model from existing data
import inspect,os
import markovify
from textstat.textstat import textstat

#Read files from directory
#Add their contents to a single string
#Make a model from this string
#generate first sentence from this model

class train():
    dir=os.path.dirname(os.path.dirname(os.path.abspath(inspect.stack()[0][1])))

    def __init__(self,line_limits=[5,7,5],file1=os.path.join(dir,"tests\Macbeth-Shakespeare.txt"),file2=os.path.join(dir,"tests\Wordsworth.txt")):
        self.line_limits=line_limits
        self.file1=file1
        self.file2=file2

    def main(self):

        textModel = train.trainOnFiles(self)
        haiku=train.genHaiku(self,textModel)

        print(haiku)

    def genHaiku(self,textModel):
        haiku=""
        for i in range(0,3):
            while True:
                sent=textModel.make_sentence()
                if(not sent==None) and (textstat.syllable_count(sent)<self.line_limits[i]):
                    haiku+=sent+"\n"
                    break

    def trainOnFiles(self):
        f1 = open(self.file1)
        text = f1.read()

        f2 = open(self.file2)
        text += f2.read()
        return markovify.NewlineText(text)

if __name__=="__main__":
    train.main(train())<|endoftext|>


<commit_before>import logging
import os

from brain import Brain

from cmdparse import Command

log = logging.getLogger("hal")

class InitCommand(Command):
    def __init__(self):
        Command.__init__(self, "init", summary="Initialize a new brain")

        self.add_option("", "--force", action="store_true")
        self.add_option("", "--order", type="int", default=5)

    def run(self, options, args):
        filename = "hal.brain"

        if os.path.exists(filename):
            if options.force:
                os.remove(filename)
            else:
                log.error("%s already exists!", filename)
                return

        Brain.init(filename, options.order)

class CloneCommand(Command):
    def __init__(self):
        Command.__init__(self, "clone", summary="Clone a MegaHAL brain")

    def run(self, options, args):
        if len(args) != 1:
            log.error("usage: clone <MegaHAL brain>")
            return

        if os.path.exists("hal.brain"):
            log.error("hal.brain already exists")
            return

        megahal_brain = args[0]

        Brain.init("hal.brain")
        b.clone(megahal_brain)
<commit_msg>Add the "learn" command<commit_after>import logging
import os

from brain import Brain

from cmdparse import Command

log = logging.getLogger("hal")

class InitCommand(Command):
    def __init__(self):
        Command.__init__(self, "init", summary="Initialize a new brain")

        self.add_option("", "--force", action="store_true")
        self.add_option("", "--order", type="int", default=5)

    def run(self, options, args):
        filename = "hal.brain"

        if os.path.exists(filename):
            if options.force:
                os.remove(filename)
            else:
                log.error("%s already exists!", filename)
                return

        Brain.init(filename, options.order)

class CloneCommand(Command):
    def __init__(self):
        Command.__init__(self, "clone", summary="Clone a MegaHAL brain")

    def run(self, options, args):
        if len(args) != 1:
            log.error("usage: clone <MegaHAL brain>")
            return

        if os.path.exists("hal.brain"):
            log.error("hal.brain already exists")
            return

        megahal_brain = args[0]

        Brain.init("hal.brain")
        b.clone(megahal_brain)

class LearnCommand(Command):
    def __init__(self):
        Command.__init__(self, "learn", summary="Learn a file of text")

    def run(self, options, args):
        if len(args) != 1:
            log.error("usage: learn <text file>")
            return

        filename = args[0]

        b = Brain("hal.brain")

        fd = open(filename)
        for line in fd.xreadlines():
            b.learn(line.strip())
<|endoftext|>


<commit_before>import copy
import hashlib
import logging
import os
import random
import struct
import time

import yaml
from yaml import CDumper

from halocoin import custom

alphabet = '123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ'

def init_logging(DEBUG, working_dir, log_file):
    if DEBUG:
        logging.basicConfig(level=logging.INFO,
                            format='%(levelname)s on %(asctime)s\n%(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S %p')
    else:
        logging.basicConfig(filename=os.path.join(working_dir, log_file),
                            level=logging.DEBUG,
                            format='%(levelname)s on %(asctime)s\n%(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S %p')
    log = logging.getLogger('werkzeug')
    log.setLevel(logging.ERROR)

def get_default_dir():
    from os.path import expanduser
    home = expanduser("~")
    default_dir = os.path.join(home, '.halocoin')
    return os.environ.get("HALOCOIN_DATA_DIR", default_dir)

def log(message):
    import traceback
    if isinstance(message, Exception):
        logging.exception(message)
        logging.critical(traceback.format_exc())
    else:
        logging.info('{}'.format(message))

def tx_owner_address(tx):
    return make_address(tx['pubkeys'], len(tx['signatures']))

def reward_owner_name(tx):
    if 'auth' in tx:
        return tx['auth']
    else:
        return get_commonname_from_certificate(tx['certificate'])

def sign(msg, privkey):
    from ecdsa import SigningKey
    if isinstance(privkey, bytes):
        privkey = SigningKey.from_string(privkey)
    return privkey.sign(msg)

def block_reward(length):
    import math
    a = length // custom.halve_at
    b = custom.block_reward / math.pow(2, a)
    return int(b)

def det_hash(x):
    """Deterministically takes sha256 of dict, list, int, or string."""
    return hashlib.sha384(yaml.dump(x).encode()).digest()[0:32]

def hash_without_nonce(block):
    a = copy.deepcopy(block)
    a.pop('nonce')
    return {'nonce': block['nonce'], 'halfHash': det_hash(a)}

def base58_encode(num):
    num = int(num.hex(), 16)
    base_count = len(alphabet)
    encode = ''
    if num < 0:
        return ''
    while num >= base_count:
        mod = num % base_count
        encode = alphabet[mod] + encode
        num = num // base_count
    if num:
        encode = alphabet[num] + encode
    return encode

def is_address_valid(address):
    if len(address) < 32:
        return False
    if not str(address[:2]).isdigit():
        return False
    if not str(address[2:]).isalnum():
        return False
    return True

def make_address(pubkeys, n):
    """
    n is the number of pubkeys required to spend from this address.
    This function is compatible with string or VerifyingKey representation of keys.
    """
    from ecdsa import VerifyingKey
    pubkeys_as_string = [p.to_string() if isinstance(p, VerifyingKey) else p for p in pubkeys]
    hashed = det_hash({str(n): pubkeys_as_string})
    return str(len(pubkeys_as_string)) + str(n) + base58_encode(hashed[0:29])

def buffer_(str_to_pad, size):
    return str_to_pad.rjust(size, '0')

def exponential_random(r, i=0):
    if random.random() < r:
        return i
    return exponential_random(r, i + 1)

def median(mylist):
    if len(mylist) < 1:
        return 0
    return sorted(mylist)[len(mylist) // 2]

def hex_sum(a, b):
    # Sum of numbers expressed as hexidecimal strings
    if isinstance(a, bytearray):
        a = a.hex()
        b = b.hex()
    return buffer_(format(int(a, 16) + int(b, 16), 'x'), 64)

def hex_invert(n):
    # Use double-size for division, to reduce information leakage.
    if isinstance(n, bytearray):
        n = n.hex()
    return buffer_(format(int('f' * 128, 16) // int(n, 16), 'x'), 64)

def encrypt(key, content, chunksize=64 * 1024):
    import io
    import Crypto.Random
    from Crypto.Cipher import AES
    infile = io.BytesIO()
    infile.write(content)
    infile.seek(0)
    outfile = io.BytesIO()
    if isinstance(key, str):
        key = key.encode()
    key = hashlib.sha256(key).digest()

    iv = Crypto.Random.OSRNG.posix.new().read(AES.block_size)
    encryptor = AES.new(key, AES.MODE_CBC, iv)
    filesize = len(content)

    outfile.write(struct.pack('<Q', filesize))
    outfile.write(iv)
    while True:
        chunk = infile.read(chunksize)
        if len(chunk) == 0:
            break
        elif len(chunk) % 16 != 0:
            chunk += '\0'.encode() * (16 - len(chunk) % 16)
        outfile.write(encryptor.encrypt(chunk))
    return outfile.getvalue()

def decrypt(key, content, chunksize=24 * 1024):
    from Crypto.Cipher import AES
    import io
    infile = io.BytesIO(content)
    outfile = io.BytesIO()

    if isinstance(key, str):
        key = key.encode()
    key = hashlib.sha256(key).digest()

    origsize = struct.unpack('<Q', infile.read(struct.calcsize('Q')))[0]
    iv = infile.read(16)
    decryptor = AES.new(key, AES.MODE_CBC, iv)

    while True:
        chunk = infile.read(chunksize)
        if len(chunk) == 0:
            break
        outfile.write(decryptor.decrypt(chunk))

    outfile.truncate(origsize)
    return outfile.getvalue()

def signature_verify(message, signature, pubkey):
    from ecdsa import VerifyingKey, SECP256k1
    if isinstance(pubkey, str):
        pubkey = VerifyingKey.from_string(pubkey, curve=SECP256k1)
    elif isinstance(pubkey, bytes):
        pubkey = VerifyingKey.from_string(pubkey, curve=SECP256k1)

    if isinstance(pubkey, VerifyingKey):
        try:
            return pubkey.verify(signature, message)
        except:
            return False
    else:
        return False

def validate_uuid4(uuid_string):

    """
    Validate that a UUID string is in
    fact a valid uuid4.
    Happily, the uuid module does the actual
    checking for us.
    It is vital that the 'version' kwarg be passed
    to the UUID() call, otherwise any 32-character
    hex string is considered valid.
    """

    try:
        from uuid import UUID
        val = UUID(uuid_string, version=4)
    except ValueError:
        # If it's a value error, then the string
        # is not a valid hex code for a UUID.
        return False

    # If the uuid_string is a valid hex code,
    # but an invalid uuid4,
    # the UUID.__init__ will convert it to a
    # valid uuid4. This is bad for validation purposes.

    return val.hex == uuid_string.replace('-', '')

def check_certificate_chain(intermediate_cert_pem):
    from OpenSSL.crypto import load_certificate, FILETYPE_PEM, X509Store, X509StoreContext
    root_cert = load_certificate(FILETYPE_PEM, custom.root_cert_pem)
    intermediate_cert = load_certificate(FILETYPE_PEM, intermediate_cert_pem)
    try:
        store = X509Store()
        store.add_cert(root_cert)
        store_ctx = X509StoreContext(store, intermediate_cert)
        store_ctx.verify_certificate()
        return True
    except:
        return False

def get_pubkey_from_certificate(intermediate_cert_pem):
    from OpenSSL.crypto import load_certificate, FILETYPE_PEM, dump_publickey
    from ecdsa import VerifyingKey
    intermediate_cert = load_certificate(FILETYPE_PEM, intermediate_cert_pem)
    pubkey_pem = dump_publickey(FILETYPE_PEM, intermediate_cert.get_pubkey())
    return VerifyingKey.from_pem(pubkey_pem)

def get_commonname_from_certificate(intermediate_cert_pem):
    from OpenSSL.crypto import load_certificate, FILETYPE_PEM
    from slugify import slugify
    intermediate_cert = load_certificate(FILETYPE_PEM, intermediate_cert_pem)
    return slugify(intermediate_cert.get_subject().commonName)

last = 0

def echo(text):
    global last
    print(text)
    last = time.time()

def techo(text):
    global last
    if last == 0:
        print(text)
    else:
        print(text + ": {}".format(time.time()-last))
        last = time.time()

def readable_bytes(num, suffix='B'):
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Yi', suffix)<commit_msg>removed submodule, reuse pyyaml<commit_after>import copy
import hashlib
import logging
import os
import random
import struct
import time

import yaml

from halocoin import custom

alphabet = '123456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ'

def init_logging(DEBUG, working_dir, log_file):
    if DEBUG:
        logging.basicConfig(level=logging.INFO,
                            format='%(levelname)s on %(asctime)s\n%(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S %p')
    else:
        logging.basicConfig(filename=os.path.join(working_dir, log_file),
                            level=logging.DEBUG,
                            format='%(levelname)s on %(asctime)s\n%(message)s',
                            datefmt='%m/%d/%Y %I:%M:%S %p')
    log = logging.getLogger('werkzeug')
    log.setLevel(logging.ERROR)

def get_default_dir():
    from os.path import expanduser
    home = expanduser("~")
    default_dir = os.path.join(home, '.halocoin')
    return os.environ.get("HALOCOIN_DATA_DIR", default_dir)

def log(message):
    import traceback
    if isinstance(message, Exception):
        logging.exception(message)
        logging.critical(traceback.format_exc())
    else:
        logging.info('{}'.format(message))

def tx_owner_address(tx):
    return make_address(tx['pubkeys'], len(tx['signatures']))

def reward_owner_name(tx):
    if 'auth' in tx:
        return tx['auth']
    else:
        return get_commonname_from_certificate(tx['certificate'])

def sign(msg, privkey):
    from ecdsa import SigningKey
    if isinstance(privkey, bytes):
        privkey = SigningKey.from_string(privkey)
    return privkey.sign(msg)

def block_reward(length):
    import math
    a = length // custom.halve_at
    b = custom.block_reward / math.pow(2, a)
    return int(b)

def det_hash(x):
    """Deterministically takes sha256 of dict, list, int, or string."""
    return hashlib.sha384(yaml.dump(x).encode()).digest()[0:32]

def hash_without_nonce(block):
    a = copy.deepcopy(block)
    a.pop('nonce')
    return {'nonce': block['nonce'], 'halfHash': det_hash(a)}

def base58_encode(num):
    num = int(num.hex(), 16)
    base_count = len(alphabet)
    encode = ''
    if num < 0:
        return ''
    while num >= base_count:
        mod = num % base_count
        encode = alphabet[mod] + encode
        num = num // base_count
    if num:
        encode = alphabet[num] + encode
    return encode

def is_address_valid(address):
    if len(address) < 32:
        return False
    if not str(address[:2]).isdigit():
        return False
    if not str(address[2:]).isalnum():
        return False
    return True

def make_address(pubkeys, n):
    """
    n is the number of pubkeys required to spend from this address.
    This function is compatible with string or VerifyingKey representation of keys.
    """
    from ecdsa import VerifyingKey
    pubkeys_as_string = [p.to_string() if isinstance(p, VerifyingKey) else p for p in pubkeys]
    hashed = det_hash({str(n): pubkeys_as_string})
    return str(len(pubkeys_as_string)) + str(n) + base58_encode(hashed[0:29])

def buffer_(str_to_pad, size):
    return str_to_pad.rjust(size, '0')

def exponential_random(r, i=0):
    if random.random() < r:
        return i
    return exponential_random(r, i + 1)

def median(mylist):
    if len(mylist) < 1:
        return 0
    return sorted(mylist)[len(mylist) // 2]

def hex_sum(a, b):
    # Sum of numbers expressed as hexidecimal strings
    if isinstance(a, bytearray):
        a = a.hex()
        b = b.hex()
    return buffer_(format(int(a, 16) + int(b, 16), 'x'), 64)

def hex_invert(n):
    # Use double-size for division, to reduce information leakage.
    if isinstance(n, bytearray):
        n = n.hex()
    return buffer_(format(int('f' * 128, 16) // int(n, 16), 'x'), 64)

def encrypt(key, content, chunksize=64 * 1024):
    import io
    import Crypto.Random
    from Crypto.Cipher import AES
    infile = io.BytesIO()
    infile.write(content)
    infile.seek(0)
    outfile = io.BytesIO()
    if isinstance(key, str):
        key = key.encode()
    key = hashlib.sha256(key).digest()

    iv = Crypto.Random.OSRNG.posix.new().read(AES.block_size)
    encryptor = AES.new(key, AES.MODE_CBC, iv)
    filesize = len(content)

    outfile.write(struct.pack('<Q', filesize))
    outfile.write(iv)
    while True:
        chunk = infile.read(chunksize)
        if len(chunk) == 0:
            break
        elif len(chunk) % 16 != 0:
            chunk += '\0'.encode() * (16 - len(chunk) % 16)
        outfile.write(encryptor.encrypt(chunk))
    return outfile.getvalue()

def decrypt(key, content, chunksize=24 * 1024):
    from Crypto.Cipher import AES
    import io
    infile = io.BytesIO(content)
    outfile = io.BytesIO()

    if isinstance(key, str):
        key = key.encode()
    key = hashlib.sha256(key).digest()

    origsize = struct.unpack('<Q', infile.read(struct.calcsize('Q')))[0]
    iv = infile.read(16)
    decryptor = AES.new(key, AES.MODE_CBC, iv)

    while True:
        chunk = infile.read(chunksize)
        if len(chunk) == 0:
            break
        outfile.write(decryptor.decrypt(chunk))

    outfile.truncate(origsize)
    return outfile.getvalue()

def signature_verify(message, signature, pubkey):
    from ecdsa import VerifyingKey, SECP256k1
    if isinstance(pubkey, str):
        pubkey = VerifyingKey.from_string(pubkey, curve=SECP256k1)
    elif isinstance(pubkey, bytes):
        pubkey = VerifyingKey.from_string(pubkey, curve=SECP256k1)

    if isinstance(pubkey, VerifyingKey):
        try:
            return pubkey.verify(signature, message)
        except:
            return False
    else:
        return False

def validate_uuid4(uuid_string):

    """
    Validate that a UUID string is in
    fact a valid uuid4.
    Happily, the uuid module does the actual
    checking for us.
    It is vital that the 'version' kwarg be passed
    to the UUID() call, otherwise any 32-character
    hex string is considered valid.
    """

    try:
        from uuid import UUID
        val = UUID(uuid_string, version=4)
    except ValueError:
        # If it's a value error, then the string
        # is not a valid hex code for a UUID.
        return False

    # If the uuid_string is a valid hex code,
    # but an invalid uuid4,
    # the UUID.__init__ will convert it to a
    # valid uuid4. This is bad for validation purposes.

    return val.hex == uuid_string.replace('-', '')

def check_certificate_chain(intermediate_cert_pem):
    from OpenSSL.crypto import load_certificate, FILETYPE_PEM, X509Store, X509StoreContext
    root_cert = load_certificate(FILETYPE_PEM, custom.root_cert_pem)
    intermediate_cert = load_certificate(FILETYPE_PEM, intermediate_cert_pem)
    try:
        store = X509Store()
        store.add_cert(root_cert)
        store_ctx = X509StoreContext(store, intermediate_cert)
        store_ctx.verify_certificate()
        return True
    except:
        return False

def get_pubkey_from_certificate(intermediate_cert_pem):
    from OpenSSL.crypto import load_certificate, FILETYPE_PEM, dump_publickey
    from ecdsa import VerifyingKey
    intermediate_cert = load_certificate(FILETYPE_PEM, intermediate_cert_pem)
    pubkey_pem = dump_publickey(FILETYPE_PEM, intermediate_cert.get_pubkey())
    return VerifyingKey.from_pem(pubkey_pem)

def get_commonname_from_certificate(intermediate_cert_pem):
    from OpenSSL.crypto import load_certificate, FILETYPE_PEM
    from slugify import slugify
    intermediate_cert = load_certificate(FILETYPE_PEM, intermediate_cert_pem)
    return slugify(intermediate_cert.get_subject().commonName)

last = 0

def echo(text):
    global last
    print(text)
    last = time.time()

def techo(text):
    global last
    if last == 0:
        print(text)
    else:
        print(text + ": {}".format(time.time()-last))
        last = time.time()

def readable_bytes(num, suffix='B'):
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f%s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Yi', suffix)<|endoftext|>


<commit_before>"Provides tools for user to load data as specified."

import os
from datetime import datetime

import pandas as pd

from batch_generator import BatchGenerator

DATASETS_PATH = os.path.join(os.environ['DEEP_QUANT_ROOT'], 'datasets')
OPEN_DF_PATH = os.path.join(DATASETS_PATH, 'open_dataset.dat')
# TODO: fix so we don't have to assume 'DEEP_QUANT_ROOT' is defined in the
# environment

def get_data_path(data_dir, filename):
    """
    Construct the data path for the experiement. If DEEP_QUANT_ROOT is
    defined in the environment, then the data path is relative to it.

    Args:
      data_dir: the directory name where experimental data is held
      filename: the data file name
    Returns:
      If DEEP_QUANT_ROOT is defined, the fully qualified data path is returned
      Otherwise a path relative to the working directory is returned
    """
    path = os.path.join(data_dir, filename)
    if data_dir != '.' and 'DEEP_QUANT_ROOT' in os.environ:
        path = os.path.join(os.environ['DEEP_QUANT_ROOT'], path)
    return path

def create_datafile(datasource, ticlist, dest_basename):
    """
    Creates a .dat file using `datasource` (either `open_dataset` or `WRDS`).
    This file will be made up of tickers from ticlist, and will reside in
    `DEEP_QUANT_ROOT`/datasets/dest_basename.

    Args:
      datasource: specifies whether the datafile should be built using
                  open_dataset as a starting point, or be built by querying
                  WRDS.
      ticlist: the name of the list of tickers that the produced .dat file
               should contain. NOTE: this should be postfixed by .dat.
      dest_basename: the name you'd like the produced datafile to have. NOTE:
                     this should be postfixed by .dat.
    """
    def get_gvkeys_from_ticlist(ticlist):  #TODO: use actual gvkeys
        """
        Returns 'gvkeys' from ticlist.dat as a sorted list.

        NOTE: Right now, 'gvkeys' are not the actual gvkeys that you'd see in
        Compustat. Instead, they're unique identifiers constructed by concatenating
        a numeric id for the exchange (1 for Nasdaq, 2 for NYSE) with the ticker
        name.
        """
        ticlist_filepath = os.path.join(DATASETS_PATH, ticlist)

        if os.path.isfile(ticlist_filepath):
            ticlist_df = pd.read_csv(ticlist_filepath, sep=' ', header=None)
            gvkeys = list()
            for line in ticlist_df.values:
                if line[1] == 'Nasdaq':
                    gvkeys.append('1'+line[0])
                elif line[1] == 'NYSE':
                    gvkeys.append('2'+line[0])
                else:
                    gvkeys.append('9'+line[0])  # TODO: is that best way to handle
                                                # unrecognized market?
        else:
            gvkeys = list()

        return gvkeys

    def shave_open_dataset(ticlist, dest):
        """
        Shaves wanted data (in terms of tics and features only; the shaving by
        dates is done by BatchGenerator's constructor), stores shaved .dat file
        at dest.

        NOTE: shaving by features not implemented yet, will rely on a
        feat_map.txt file.
        """
        gvkeys = get_gvkeys_from_ticlist(ticlist)
        open_df = pd.read_csv(OPEN_DF_PATH, sep=' ', dtype={'gvkey': str})
        shaved_df = open_df[open_df.gvkey.isin(gvkeys)]
        shaved_df.to_csv(dest, sep=' ', index=False)

    def write_WRDS_data(dest):
        """
        Writes .dat file using data from WRDS.
        """
        raise NotImplementedError("Sorry! WRDS integration not ready.")  # TODO

    dest = get_data_path(DATASETS_PATH, dest_basename)

    if datasource == "open_dataset":
        shave_open_dataset(ticlist, dest)
    elif datasource == "WRDS":
        write_WRDS_data(ticlist, dest)
    else:
        raise Exception("Unknown datasource.")

def load_all_data(config):
    """
    Returns all data as a BatchGenerator object.
    """
    data_path = get_data_path(config.data_dir, config.datafile)
    batches = BatchGenerator(data_path, config)

    return batches

def load_train_valid_data(config):
    """
    Returns train_data and valid_data, both as BatchGenerator objects.
    """
    batches = load_all_data(config)

    train_data = batches.train_batches()
    valid_data = batches.valid_batches()

    return train_data, valid_data
<commit_msg>20X+ speedup in caching batches :-)<commit_after>"Provides tools for user to load data as specified."

import os
from datetime import datetime

import pandas as pd

from batch_generator_new import BatchGenerator

DATASETS_PATH = os.path.join(os.environ['DEEP_QUANT_ROOT'], 'datasets')
OPEN_DF_PATH = os.path.join(DATASETS_PATH, 'open_dataset.dat')
# TODO: fix so we don't have to assume 'DEEP_QUANT_ROOT' is defined in the
# environment

def get_data_path(data_dir, filename):
    """
    Construct the data path for the experiement. If DEEP_QUANT_ROOT is
    defined in the environment, then the data path is relative to it.

    Args:
      data_dir: the directory name where experimental data is held
      filename: the data file name
    Returns:
      If DEEP_QUANT_ROOT is defined, the fully qualified data path is returned
      Otherwise a path relative to the working directory is returned
    """
    path = os.path.join(data_dir, filename)
    if data_dir != '.' and 'DEEP_QUANT_ROOT' in os.environ:
        path = os.path.join(os.environ['DEEP_QUANT_ROOT'], path)
    return path

def create_datafile(datasource, ticlist, dest_basename):
    """
    Creates a .dat file using `datasource` (either `open_dataset` or `WRDS`).
    This file will be made up of tickers from ticlist, and will reside in
    `DEEP_QUANT_ROOT`/datasets/dest_basename.

    Args:
      datasource: specifies whether the datafile should be built using
                  open_dataset as a starting point, or be built by querying
                  WRDS.
      ticlist: the name of the list of tickers that the produced .dat file
               should contain. NOTE: this should be postfixed by .dat.
      dest_basename: the name you'd like the produced datafile to have. NOTE:
                     this should be postfixed by .dat.
    """
    def get_gvkeys_from_ticlist(ticlist):  #TODO: use actual gvkeys
        """
        Returns 'gvkeys' from ticlist.dat as a sorted list.

        NOTE: Right now, 'gvkeys' are not the actual gvkeys that you'd see in
        Compustat. Instead, they're unique identifiers constructed by concatenating
        a numeric id for the exchange (1 for Nasdaq, 2 for NYSE) with the ticker
        name.
        """
        ticlist_filepath = os.path.join(DATASETS_PATH, ticlist)

        if os.path.isfile(ticlist_filepath):
            ticlist_df = pd.read_csv(ticlist_filepath, sep=' ', header=None)
            gvkeys = list()
            for line in ticlist_df.values:
                if line[1] == 'Nasdaq':
                    gvkeys.append('1'+line[0])
                elif line[1] == 'NYSE':
                    gvkeys.append('2'+line[0])
                else:
                    gvkeys.append('9'+line[0])  # TODO: is that best way to handle
                                                # unrecognized market?
        else:
            gvkeys = list()

        return gvkeys

    def shave_open_dataset(ticlist, dest):
        """
        Shaves wanted data (in terms of tics and features only; the shaving by
        dates is done by BatchGenerator's constructor), stores shaved .dat file
        at dest.

        NOTE: shaving by features not implemented yet, will rely on a
        feat_map.txt file.
        """
        gvkeys = get_gvkeys_from_ticlist(ticlist)
        open_df = pd.read_csv(OPEN_DF_PATH, sep=' ', dtype={'gvkey': str})
        shaved_df = open_df[open_df.gvkey.isin(gvkeys)]
        shaved_df.to_csv(dest, sep=' ', index=False)

    def write_WRDS_data(dest):
        """
        Writes .dat file using data from WRDS.
        """
        raise NotImplementedError("Sorry! WRDS integration not ready.")  # TODO

    dest = get_data_path(DATASETS_PATH, dest_basename)

    if datasource == "open_dataset":
        shave_open_dataset(ticlist, dest)
    elif datasource == "WRDS":
        write_WRDS_data(ticlist, dest)
    else:
        raise Exception("Unknown datasource.")

def load_all_data(config):
    """
    Returns all data as a BatchGenerator object.
    """
    data_path = get_data_path(config.data_dir, config.datafile)
    batches = BatchGenerator(data_path, config)

    return batches

def load_train_valid_data(config):
    """
    Returns train_data and valid_data, both as BatchGenerator objects.
    """
    batches = load_all_data(config)

    train_data = batches.train_batches()
    valid_data = batches.valid_batches()

    return train_data, valid_data
<|endoftext|>


<commit_before>from telegram.ext import CommandHandler, Filters

from classes import User
from database import QuoteDatabase
from main import database, username

LOUDLY_CRYING_FACE = L = '\U0001f62d'

def handle_addquote(bot, update, word='quote'):
    message = update.message
    quote = update.message.reply_to_message

    assert quote is not None

    # Only text messages can be added as quotes
    if (quote.photo is not None or quote.video is not None) and quote.caption:
        text = quote.caption
        text_html = text
    elif quote.text is not None:
        text = quote.text
        text_html = quote.text_html
    else:
        return

    chat_id = message.chat.id
    message_id = quote.message_id
    quoted_by = message.from_user

    # Forwarded messages
    is_forward = quote.forward_from is not None

    if is_forward:
        sent_by = quote.forward_from
        sent_at = quote.forward_date.timestamp()
    else:
        sent_by = quote.from_user
        sent_at = quote.date.timestamp()

    # Bot messages can't be added as quotes
    if sent_by.username == username.lstrip('@'):
        response = f"can't {word} bot messages"
        return update.message.reply_text(response)

    # Users can't add their own messages as quotes
    if sent_by.id == quoted_by.id:
        response = f"can't {word} own messages"
        return update.message.reply_text(response)

    database.add_or_update_user(User.from_telegram(sent_by))
    database.add_or_update_user(User.from_telegram(quoted_by))

    result, status = database.add_quote(
        chat_id, message_id, is_forward,
        sent_at, sent_by.id, text, text_html, quoted_by.id)

    if status == database.QUOTE_ADDED:
        response = f"{word} added"
    elif status == database.QUOTE_ALREADY_EXISTS:
        response = f"{word} already exists"
    elif status == database.QUOTE_PREVIOUSLY_DELETED:
        response = f"{word} was previously deleted"
        return update.message.reply_text(response)

    message = update.message.reply_text(response)

    database.add_message(message.chat_id, message.message_id, result.quote.id)

def handle_addqoute(bot, update):
    return handle_addquote(bot, update, word='qoute')

def handle_sadquote(bot, update):
    return handle_addquote(bot, update, word=f'{L} quote {L}')

handler_addquote = CommandHandler(
    'addquote', handle_addquote, filters=Filters.reply & Filters.group)

handler_addqoute = CommandHandler(
    'addqoute', handle_addqoute, filters=Filters.reply & Filters.group)

handler_sadquote = CommandHandler(
    'sadquote', handle_sadquote, filters=Filters.reply & Filters.group)
<commit_msg>Add another 'addquote' alias<commit_after>from telegram.ext import CommandHandler, Filters

from classes import User
from database import QuoteDatabase
from main import database, username

LOUDLY_CRYING_FACE = '\U0001F62D'
ANGRY_FACE = '\U0001F620'
POUTING_FACE = '\U0001F621'
SMILING_FACE_WITH_SUNGLASSES = '\U0001F60E'

def format_response(s, emoji):
    if emoji is not None:
        return f' {emoji} '.join(s.split(' '))
    return s

def handle_addquote(bot, update, word='quote', emoji=None):
    message = update.message
    quote = update.message.reply_to_message

    assert quote is not None

    # Only text messages or media captions can be quoted
    if (quote.photo is not None or quote.video is not None) and quote.caption:
        text = quote.caption
        text_html = text
    elif quote.text is not None:
        text = quote.text
        text_html = quote.text_html
    else:
        return

    chat_id = message.chat.id
    message_id = quote.message_id
    quoted_by = message.from_user

    # Forwarded messages
    is_forward = quote.forward_from is not None

    if is_forward:
        sent_by = quote.forward_from
        sent_at = quote.forward_date.timestamp()
    else:
        sent_by = quote.from_user
        sent_at = quote.date.timestamp()

    # Bot messages can't be added as quotes
    if sent_by.username == username.lstrip('@'):
        response = format_response(f"can't {word} soup messages", emoji)
        return update.message.reply_text(response)

    # Users can't add their own messages as quotes
    if sent_by.id == quoted_by.id:
        response = format_response(f"can't {word} your own messages", emoji)
        return update.message.reply_text(response)

    database.add_or_update_user(User.from_telegram(sent_by))
    database.add_or_update_user(User.from_telegram(quoted_by))

    result, status = database.add_quote(
        chat_id, message_id, is_forward,
        sent_at, sent_by.id, text, text_html, quoted_by.id)

    if status == database.QUOTE_ADDED:
        response = f"{word} added"
    elif status == database.QUOTE_ALREADY_EXISTS:
        response = f"{word} already exists"
    elif status == database.QUOTE_PREVIOUSLY_DELETED:
        response = format_response(f"this {word} was previously deleted", emoji)
        return update.message.reply_text(response)

    response = format_response(response, emoji)
    message = update.message.reply_text(response)

    database.add_message(message.chat_id, message.message_id, result.quote.id)

def handle_addqoute(bot, update):
    return handle_addquote(bot, update, word='qoute')

def handle_sadquote(bot, update):
    return handle_addquote(bot, update, emoji=LOUDLY_CRYING_FACE)

def handle_madquote(bot, update):
    return handle_addquote(bot, update, emoji=POUTING_FACE)

def handle_radquote(bot, update):
    return handle_addquote(bot, update, emoji=SMILING_FACE_WITH_SUNGLASSES)

handler_addquote = CommandHandler(
    'addquote', handle_addquote, filters=Filters.reply & Filters.group)

handler_addqoute = CommandHandler(
    'addqoute', handle_addqoute, filters=Filters.reply & Filters.group)

handler_sadquote = CommandHandler(
    'sadquote', handle_sadquote, filters=Filters.reply & Filters.group)

handler_madquote = CommandHandler(
    'madquote', handle_madquote, filters=Filters.reply & Filters.group)
<|endoftext|>


<commit_before>
from os.path import exists
from shutil import copy
from json import load, dump

def __init__(self):
    self.logger.info("Starting statistics engine.")
    if exists('data/stats.json'):
        self.logger.info("Found statistics file.")
    else:
        self.logger.info("Statistics file not found. Creating.")
        copy('data/stats-template.json')
        self.logger.info("Created!")

    self.logger.info("Started statistics engine.")

def add_resub(username):
    with open('data/stats.json') as f:
        stats = load(f)
        stats['total-resubs'] += 1
        dump(stats, f)

def add_sub(username):
        with open('data/stats.json') as f:
            stats = load(f)
            stats['total-subs'] += 1
            dump(stats, f)

def add_follower(username):
        with open('data/stats.json') as f:
            stats = load(f)
            stats['total-followers'] += 1
            dump(stats, f)

def add_total_message(username):
        with open('data/stats.json') as f:
            stats = load(f)
            stats['total-followers'] += 1
            dump(stats, f)

def add_total_view(username):
        with open('data/stats.json') as f:
            stats = load(f)
            stats['total-views'] += 1
            dump(stats, f)

def add_unique_viewer(username):
    with open('data/stats.json') as f:
        stats = load(f)
        stats['unique-views'] += 1
        dump(stats, f)

def add_deleted_message(username):

def add_banned_user(username):
    with open('data/stats.json') as f:
        stats = load(f)
        stats['total-banned-users'] += 1
        dump(stats, f)
<commit_msg>Remove unneeded file<commit_after><|endoftext|>


<commit_before>#!/usr/local/bin/python2.7

"""
    Copyright (c) 2017 Ad Schellevis <ad@opnsense.org>
    All rights reserved.

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are met:

    1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.

    2. Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.

    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
    POSSIBILITY OF SUCH DAMAGE.

    --------------------------------------------------------------------------------------
    read filter log, limit by number of records or last received digest (md5 hash of row)
"""
import os
import sys
import md5
import argparse
import ujson
sys.path.insert(0, "/usr/local/opnsense/site-python")
from log_helper import reverse_log_reader, fetch_clog
from params import update_params

filter_log = '/var/log/filter.log'

# define log layouts, every endpoint contains all options
# source : https://github.com/opnsense/ports/blob/master/opnsense/filterlog/files/description.txt
fields_general = 'rulenr,subrulenr,anchorname,ridentifier,interface,reason,action,dir,version'.split(',')
fields_ipv4 = fields_general + 'tos,ecn,ttl,id,offset,flags,proto,protoname,length,src,dst'.split(',')
fields_ipv4_udp = fields_ipv4 + 'srcport,dstport,datalen'.split(',')
fields_ipv4_tcp = fields_ipv4 + 'srcport,dstport,datalen,flags,error_options'.split(',')
fields_ipv4_carp = fields_ipv4 + 'type,ttl,vhid,version,advskew,advbase'.split(',')

fields_ipv6 = fields_general + 'class,flowlabel,hlim,next-header,next,payload-length,src,dst'.split(',')
fields_ipv6_udp = fields_ipv6 + 'srcport,dstport,datalen'.split(',')
fields_ipv6_tcp = fields_ipv6 + 'srcport,dstport,datalen,flags,error_options'.split(',')
fields_ipv6_carp = fields_ipv6 + 'type,ttl,vhid,version2,advskew,advbase'.split(',')

def update_rule(target, ruleparts, spec):
    """ update target rule with parts in spec
        :param target: target rule
        :param ruleparts: list of rule items
        :param spec: full rule specification, depending on protocol and version
    """
    while len(target) < len(spec) and len(ruleparts) > 0:
        target[spec[len(target)]] = ruleparts.pop(0)
    # full spec
    target['__spec__'] = spec

if __name__ == '__main__':
    # read parameters
    parameters = {'limit': '0', 'digest': ''}
    update_params(parameters)
    parameters['limit'] = int(parameters['limit'])

    result = list()
    for record in reverse_log_reader(fetch_clog(filter_log)):
        if record['line'].find('filterlog') > -1:
            rule = dict()
            rulep = record['line'].split('filterlog:')[1].strip().split(',')
            update_rule(rule, rulep, fields_general)

            if 'version' in rule:
                if rule['version'] == '4':
                    update_rule(rule, rulep, fields_ipv4)
                    if 'proto' in rule:
                        if rule['proto'] == '17': # UDP
                            update_rule(rule, rulep, fields_ipv4_udp)
                        elif  rule['proto'] == '6': # TCP
                            update_rule(rule, rulep, fields_ipv4_tcp)
                        elif  rule['proto'] == '112': # CARP
                            update_rule(rule, rulep, fields_ipv4_carp)
                elif rule['version'] == '6':
                    update_rule(rule, rulep, fields_ipv6)
                    if 'next' in rule:
                        if rule['next'] == '17': # UDP
                            update_rule(rule, rulep, fields_ipv6_udp)
                        elif  rule['next'] == '6': # TCP
                            update_rule(rule, rulep, fields_ipv6_tcp)
                        elif  rule['next'] == '112': # CARP
                            update_rule(rule, rulep, fields_ipv6_carp)

            result.append(rule)

            # rule metadata (unique hash, hostname, timestamp)
            tmp = record['line'].split('filterlog:')[0].split()
            rule['__digest__'] = md5.new(record['line']).hexdigest()
            rule['__host__'] = tmp.pop()
            rule['__timestamp__'] =  ' '.join(tmp)

            # handle exit criteria, row limit or last digest
            if parameters['limit'] != 0 and len(result) > parameters['limit']:
                break
            elif parameters['digest'].strip() != '' and parameters['digest'] == rule['__digest__']:
                break

    print (ujson.dumps(result))
<commit_msg>firewall log parser script, bugfix, split rule and metadata during processing. for https://github.com/opnsense/core/issues/1788<commit_after>#!/usr/local/bin/python2.7

"""
    Copyright (c) 2017 Ad Schellevis <ad@opnsense.org>
    All rights reserved.

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are met:

    1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.

    2. Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.

    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
    POSSIBILITY OF SUCH DAMAGE.

    --------------------------------------------------------------------------------------
    read filter log, limit by number of records or last received digest (md5 hash of row)
"""
import os
import sys
import md5
import argparse
import ujson
sys.path.insert(0, "/usr/local/opnsense/site-python")
from log_helper import reverse_log_reader, fetch_clog
from params import update_params

filter_log = '/var/log/filter.log'

# define log layouts, every endpoint contains all options
# source : https://github.com/opnsense/ports/blob/master/opnsense/filterlog/files/description.txt
fields_general = 'rulenr,subrulenr,anchorname,ridentifier,interface,reason,action,dir,version'.split(',')
fields_ipv4 = fields_general + 'tos,ecn,ttl,id,offset,flags,proto,protoname,length,src,dst'.split(',')
fields_ipv4_udp = fields_ipv4 + 'srcport,dstport,datalen'.split(',')
fields_ipv4_tcp = fields_ipv4 + 'srcport,dstport,datalen,flags,error_options'.split(',')
fields_ipv4_carp = fields_ipv4 + 'type,ttl,vhid,version,advskew,advbase'.split(',')

fields_ipv6 = fields_general + 'class,flowlabel,hlim,next-header,next,payload-length,src,dst'.split(',')
fields_ipv6_udp = fields_ipv6 + 'srcport,dstport,datalen'.split(',')
fields_ipv6_tcp = fields_ipv6 + 'srcport,dstport,datalen,flags,error_options'.split(',')
fields_ipv6_carp = fields_ipv6 + 'type,ttl,vhid,version2,advskew,advbase'.split(',')

def update_rule(target, metadata_target, ruleparts, spec):
    """ update target rule with parts in spec
        :param target: target rule
        :param metadata_target: collected metadata
        :param ruleparts: list of rule items
        :param spec: full rule specification, depending on protocol and version
    """
    while len(target) < len(spec) and len(ruleparts) > 0:
        target[spec[len(target)]] = ruleparts.pop(0)
    # full spec
    metadata_target['__spec__'] = spec

if __name__ == '__main__':
    # read parameters
    parameters = {'limit': '0', 'digest': ''}
    update_params(parameters)
    parameters['limit'] = int(parameters['limit'])

    result = list()
    for record in reverse_log_reader(fetch_clog(filter_log)):
        if record['line'].find('filterlog') > -1:
            rule = dict()
            metadata = dict()
            # rule metadata (unique hash, hostname, timestamp)
            tmp = record['line'].split('filterlog:')[0].split()
            metadata['__digest__'] = md5.new(record['line']).hexdigest()
            metadata['__host__'] = tmp.pop()
            metadata['__timestamp__'] =  ' '.join(tmp)
            rulep = record['line'].split('filterlog:')[1].strip().split(',')
            update_rule(rule, metadata, rulep, fields_general)

            if 'version' in rule:
                if rule['version'] == '4':
                    update_rule(rule, metadata, rulep, fields_ipv4)
                    if 'proto' in rule:
                        if rule['proto'] == '17': # UDP
                            update_rule(rule, metadata, rulep, fields_ipv4_udp)
                        elif  rule['proto'] == '6': # TCP
                            update_rule(rule, metadata, rulep, fields_ipv4_tcp)
                        elif  rule['proto'] == '112': # CARP
                            update_rule(rule, metadata, rulep, fields_ipv4_carp)
                elif rule['version'] == '6':
                    update_rule(rule, metadata, rulep, fields_ipv6)
                    if 'next' in rule:
                        if rule['next'] == '17': # UDP
                            update_rule(rule, metadata, rulep, fields_ipv6_udp)
                        elif  rule['next'] == '6': # TCP
                            update_rule(rule, metadata, rulep, fields_ipv6_tcp)
                        elif  rule['next'] == '112': # CARP
                            update_rule(rule, metadata, rulep, fields_ipv6_carp)

            rule.update(metadata)
            result.append(rule)

            # handle exit criteria, row limit or last digest
            if parameters['limit'] != 0 and len(result) > parameters['limit']:
                break
            elif parameters['digest'].strip() != '' and parameters['digest'] == rule['__digest__']:
                break

    print (ujson.dumps(result))
<|endoftext|>


<commit_before>#!/usr/bin/python3.4.2
'''
Created on Apr 6, 2015

Name something that might be worn on the foot.

Change one letter in it without changing the
order of the other letters. The result
will name something one might wear on the
upper part of the body. What is it?

Here's a hint: The thing on the upper part
of the body is a two-word phrase.

@author: john.obrien, leiran.biton
'''

import json
from string import ascii_lowercase as alphabet
from nltk.corpus import words, wordnet
import nltk

'''
search
parameters:
1. a dictionary where the keys are words
in English and the values are the definitions
of words
2. a list of keywords which we are trying to find
in the definitions of the words

returns:
a list of words for which the keywords searched for
were found in the definitions
'''

def search(d, kws):
    matches = []
    # check kws against all defs in wordnets list of words
    for word in words.words():
        if type(word) == str:
            synset = wordnet.synsets(word)
            for syn in synset:
                if all([kw.lower() in syn.definition().lower() for kw in kws]):
                            matches.append(word.lower())
    return matches

def build_candidates(footwear, upperwear):
    print("Candidates-")
    candidates = []
    for word in footwear:
        for offset, letter in enumerate(word):
            for new_letter in list(set(alphabet).difference(letter)):
                new_word = word[:offset] + new_letter + word[offset+1:]
                if new_word in upperwear:
                    candidates.append((word, new_word))
                    print("    Original Word: {0}-> Changed Word:{1}".format(word, new_word))
    return candidates

if __name__ == "__main__":
    nltk.download('wordnet')
    nltk.download('words')
    print("Running test case...")
    wd = {"boot": "worn Foot",
          "Coot": "worn upper"}
    footwear   = search(wd, ["worn", "foot"])
    upperwear  = search(wd, ["worn", "upper"])
    candidates = build_candidates(footwear, upperwear)
    library = "dictionary.json"
    print("Running with library {0}".format(library))
    with open (library, "r") as fp:
        wd = json.load(fp)
    footwear   = search(wd, ["worn", "foot"])
    upperwear  = search(wd, ["worn", "upper"])
    candidates = build_candidates(footwear, upperwear)
<commit_msg>Added in Leiran's suggested changes to use NLTK as the dictionary.<commit_after>#!/usr/bin/python3.4.2
'''
Created on Apr 6, 2015

Name something that might be worn on the foot.

Change one letter in it without changing the
order of the other letters. The result
will name something one might wear on the
upper part of the body. What is it?

Here's a hint: The thing on the upper part
of the body is a two-word phrase.

@author: john.obrien, leiran.biton
'''

import json
from string import ascii_lowercase as alphabet
from nltk.corpus import words, wordnet
import nltk

'''
search
parameters:
1. a dictionary where the keys are words
in English and the values are the definitions
of words
2. a list of keywords which we are trying to find
in the definitions of the words

returns:
a list of words for which the keywords searched for
were found in the definitions
'''

def search(kws):
    matches = []
    # check kws against all defs in wordnets list of words
    for word in str(words.words()):
            synset = wordnet.synsets(word)
            for syn in synset:
                if all([kw.lower() in syn.definition().lower() for kw in kws]):
                            matches.append(word.lower())
    return matches

def build_candidates(footwear, upperwear):
    print("Candidates-")
    candidates = []
    for word in footwear:
        for offset, letter in enumerate(word):
            for new_letter in list(set(alphabet).difference(letter)):
                new_word = word[:offset] + new_letter + word[offset+1:]
                if new_word in upperwear:
                    candidates.append((word, new_word))
                    print("    Original Word: {0}-> Changed Word:{1}".format(word, new_word))
    return candidates

if __name__ == "__main__":
    nltk.download('wordnet')
    nltk.download('words')
#    print("Running test case...")
#    wd = {"boot": "worn Foot",
#         "Coot": "worn upper"}
#    footwear   = search(wd, ["worn", "foot"])
#    upperwear  = search(wd, ["worn", "upper"])
#    candidates = build_candidates(footwear, upperwear)
#    library = "dictionary.json"
#    print("Running with library {0}".format(library))
#    with open (library, "r") as fp:
#        wd = json.load(fp)
    footwear   = search(["worn", "foot"])
    upperwear  = search(["worn", "upper"])
    candidates = build_candidates(footwear, upperwear)
<|endoftext|>