<commit_before># Copyright (C) 2010-2013 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import time
import shutil
import logging
import Queue
from threading import Thread, Lock

from lib.cuckoo.common.constants import CUCKOO_ROOT
from lib.cuckoo.common.exceptions import CuckooMachineError, CuckooGuestError, CuckooOperationalError, CuckooCriticalError
from lib.cuckoo.common.objects import File
from lib.cuckoo.common.utils import create_folder
from lib.cuckoo.common.config import Config
from lib.cuckoo.core.database import Database, TASK_COMPLETED, TASK_REPORTED
from lib.cuckoo.core.guest import GuestManager
from lib.cuckoo.core.resultserver import Resultserver
from lib.cuckoo.core.plugins import list_plugins, RunAuxiliary, RunProcessing, RunSignatures, RunReporting

log = logging.getLogger(__name__)

machinery = None
machine_lock = Lock()

total_analysis_count = 0
active_analysis_count = 0

class AnalysisManager(Thread):
    """Analysis Manager.

    This class handles the full analysis process for a given task. It takes
    care of selecting the analysis machine, preparing the configuration and
    interacting with the guest agent and analyzer components to launch and
    complete the analysis and store, process and report its results.
    """

    def __init__(self, task, error_queue):
        """@param task: task object containing the details for the analysis."""
        Thread.__init__(self)
        Thread.daemon = True

        self.task = task
        self.errors = error_queue
        self.cfg = Config()
        self.storage = ""
        self.binary = ""
        self.machine = None

    def init_storage(self):
        """Initialize analysis storage folder."""
        self.storage = os.path.join(CUCKOO_ROOT,
                                    "storage",
                                    "analyses",
                                    str(self.task.id))

        # If the analysis storage folder already exists, we need to abort the
        # analysis or previous results will be overwritten and lost.
        if os.path.exists(self.storage):
            log.error("Analysis results folder already exists at path \"%s\","
                      " analysis aborted", self.storage)
            return False

        # If we're not able to create the analysis storage folder, we have to
        # abort the analysis.
        try:
            create_folder(folder=self.storage)
        except CuckooOperationalError:
            log.error("Unable to create analysis folder %s", self.storage)
            return False

        return True

    def store_file(self):
        """Store a copy of the file being analyzed."""
        if not os.path.exists(self.task.target):
            log.error("The file to analyze does not exist at path \"%s\", "
                      "analysis aborted", self.task.target)
            return False

        sha256 = File(self.task.target).get_sha256()
        self.binary = os.path.join(CUCKOO_ROOT, "storage", "binaries", sha256)

        if os.path.exists(self.binary):
            log.info("File already exists at \"%s\"", self.binary)
        else:
            # TODO: do we really need to abort the analysis in case we are not
            # able to store a copy of the file?
            try:
                shutil.copy(self.task.target, self.binary)
            except (IOError, shutil.Error) as e:
                log.error("Unable to store file from \"%s\" to \"%s\", "
                          "analysis aborted", self.task.target, self.binary)
                return False

        try:
            new_binary_path = os.path.join(self.storage, "binary")

            if hasattr(os, "symlink"):
                os.symlink(self.binary, new_binary_path)
            else:
                shutil.copy(self.binary, new_binary_path)
        except (AttributeError, OSError) as e:
            log.error("Unable to create symlink/copy from \"%s\" to \"%s\"", self.binary, self.storage)

        return True

    def acquire_machine(self):
        """Acquire an analysis machine from the pool of available ones."""
        machine = None

        # Start a loop to acquire the a machine to run the analysis on.
        while True:
            machine_lock.acquire()
            # If the user specified a specific machine ID, a platform to be
            # used or machine tags acquire the machine accordingly.
            try:
                machine = machinery.acquire(machine_id=self.task.machine,
                                            platform=self.task.platform,
                                            tags=self.task.tags)
            finally:
                machine_lock.release()

            # If no machine is available at this moment, wait for one second
            # and try again.
            if not machine:
                log.debug("Task #%d: no machine available yet", self.task.id)
                time.sleep(1)
            else:
                log.info("Task #%d: acquired machine %s (label=%s)", self.task.id, machine.name, machine.label)
                break

        self.machine = machine

    def build_options(self):
        """Generate analysis options.
        @return: options dict.
        """
        options = {}

        options["id"] = self.task.id
        options["ip"] = self.machine.resultserver_ip
        options["port"] = self.machine.resultserver_port
        options["category"] = self.task.category
        options["target"] = self.task.target
        options["package"] = self.task.package
        options["options"] = self.task.options
        options["enforce_timeout"] = self.task.enforce_timeout
        options["clock"] = self.task.clock

        if not self.task.timeout or self.task.timeout == 0:
            options["timeout"] = self.cfg.timeouts.default
        else:
            options["timeout"] = self.task.timeout

        if self.task.category == "file":
            options["file_name"] = File(self.task.target).get_name()
            options["file_type"] = File(self.task.target).get_type()

        return options

    def launch_analysis(self):
        """Start analysis."""
        succeeded = False

        log.info("Starting analysis of %s \"%s\" (task=%d)", self.task.category.upper(), self.task.target, self.task.id)

        # Initialize the the analysis folders.
        if not self.init_storage():
            return False

        if self.task.category == "file":
            # Store a copy of the original file.
            if not self.store_file():
                return False

        # Acquire analysis machine.
        try:
            self.acquire_machine()
        except CuckooOperationalError as e:
            log.error("Cannot acquire machine: {0}".format(e))
            return False

        # Generate the analysis configuration file.
        options = self.build_options()

        # At this point we can tell the Resultserver about it.
        try:
            Resultserver().add_task(self.task, self.machine)
        except Exception as e:
            machinery.release(self.machine.label)
            self.errors.put(e)

        aux = RunAuxiliary(task=self.task, machine=self.machine)
        aux.start()

        try:
            # Mark the selected analysis machine in the database as started.
            guest_log = Database().guest_start(self.task.id,
                                               self.machine.name,
                                               self.machine.label,
                                               machinery.__class__.__name__)
            # Start the machine.
            machinery.start(self.machine.label)
        except CuckooMachineError as e:
            log.error(str(e), extra={"task_id": self.task.id})

            # Stop Auxiliary modules.
            aux.stop()

            return False
        else:
            try:
                # Initialize the guest manager.
                guest = GuestManager(self.machine.name, self.machine.ip, self.machine.platform)
                # Start the analysis.
                guest.start_analysis(options)
            except CuckooGuestError as e:
                log.error(str(e), extra={"task_id": self.task.id})

                # Stop Auxiliary modules.
                aux.stop()

                return False
            else:
                # Wait for analysis completion.
                try:
                    guest.wait_for_completion()
                    succeeded = True
                except CuckooGuestError as e:
                    log.error(str(e), extra={"task_id": self.task.id})
                    succeeded = False

        finally:
            # Stop Auxiliary modules.
            aux.stop()

            # Take a memory dump of the machine before shutting it off.
            if self.cfg.cuckoo.memory_dump or self.task.memory:
                try:
                    machinery.dump_memory(self.machine.label,
                                          os.path.join(self.storage, "memory.dmp"))
                except NotImplementedError:
                    log.error("The memory dump functionality is not available "
                              "for current machine manager")
                except CuckooMachineError as e:
                    log.error(e)

            try:
                # Stop the analysis machine.
                machinery.stop(self.machine.label)
            except CuckooMachineError as e:
                log.warning("Unable to stop machine %s: %s", self.machine.label, e)

            # Market the machine in the database as stopped.
            Database().guest_stop(guest_log)

            try:
                # Release the analysis machine.
                machinery.release(self.machine.label)
            except CuckooMachineError as e:
                log.error("Unable to release machine %s, reason %s. "
                          "You might need to restore it manually", self.machine.label, e)

            # after all this, we can make the Resultserver forget about it
            Resultserver().del_task(self.task, self.machine)

        return succeeded

    def process_results(self):
        """Process the analysis results and generate the enabled reports."""
        results = RunProcessing(task_id=self.task.id).run()
        RunSignatures(results=results).run()
        RunReporting(task_id=self.task.id, results=results).run()

        for proc in results["behavior"]["processes"]:
            log.debug("ParseProcessLog instance for %d (%s) parsed its log %d times.",
                proc["process_id"], proc["process_name"], proc["calls"].parsecount)

        # If the target is a file and the user enabled the option,
        # delete the original copy.
        if self.task.category == "file" and self.cfg.cuckoo.delete_original:
            try:
                os.remove(self.task.target)
            except OSError as e:
                log.error("Unable to delete original file at path \"%s\": %s", self.task.target, e)

        log.info("Task #%d: reports generation completed (path=%s)", self.task.id, self.storage)

        return True

    def run(self):
        """Run manager thread."""
        global active_analysis_count, total_analysis_count
        active_analysis_count += 1
        success = self.launch_analysis()
        Database().set_status(self.task.id, TASK_COMPLETED)

        log.debug("Released database task #%d with status %s", self.task.id, success)

        self.process_results()
        Database().set_status(self.task.id, TASK_REPORTED)

        log.info("Task #%d: analysis procedure completed", self.task.id)
        active_analysis_count -= 1

class Scheduler:
    """Tasks Scheduler.

    This class is responsible for the main execution loop of the tool. It
    prepares the analysis machines and keep waiting and loading for new
    analysis tasks.
    Whenever a new task is available, it launches AnalysisManager which will
    take care of running the full analysis process and operating with the
    assigned analysis machine.
    """

    def __init__(self):
        self.running = True
        self.cfg = Config()
        self.db = Database()

    def initialize(self):
        """Initialize the machine manager."""
        global machinery

        machinery_name = self.cfg.cuckoo.machine_manager

        log.info("Using \"%s\" machine manager", machinery_name)

        # Get registered class name. Only one machine manager is imported,
        # therefore there should be only one class in the list.
        plugin = list_plugins("machinery")[0]
        # Initialize the machine manager.
        machinery = plugin()

        # Find its configuration file.
        conf = os.path.join(CUCKOO_ROOT, "conf", "%s.conf" % machinery_name)

        if not os.path.exists(conf):
            raise CuckooCriticalError("The configuration file for machine "
                                      "manager \"{0}\" does not exist at path: "
                                      "{1}".format(machinery_name, conf))

        # Provide a dictionary with the configuration options to the
        # machine manager instance.
        machinery.set_options(Config(conf))
        # Initialize the machine manager.
        machinery.initialize(machinery_name)

        # At this point all the available machines should have been identified
        # and added to the list. If none were found, Cuckoo needs to abort the
        # execution.
        if len(machinery.machines()) == 0:
            raise CuckooCriticalError("No machines available")
        else:
            log.info("Loaded %s machine/s", len(machinery.machines()))

    def stop(self):
        """Stop scheduler."""
        self.running = False
        # Shutdown machine manager (used to kill machines that still alive).
        machinery.shutdown()

    def start(self):
        """Start scheduler."""
        self.initialize()

        log.info("Waiting for analysis tasks...")

        # Message queue with threads to transmit exceptions (used as IPC).
        errors = Queue.Queue()

        maxcount = self.cfg.cuckoo.max_analysis_count

        # This loop runs forever.
        while self.running:
            time.sleep(1)

            # If no machines are available, it's pointless to fetch for
            # pending tasks. Loop over.
            if machinery.availables() == 0:
                continue

            if total_analysis_count >= maxcount:
                if active_analysis_count <= 0:
                    self.stop()
            else:
                # Fetch a pending analysis task.
                task = self.db.fetch()

                if task:
                    log.debug("Processing task #%s", task.id)
                    total_analysis_count += 1

                    # Initialize the analysis manager.
                    analysis = AnalysisManager(task, errors)
                    # Start.
                    analysis.start()

            # Deal with errors.
            try:
                error = errors.get(block=False)
            except Queue.Empty:
                pass
            else:
                raise error
<commit_msg>revert my life pls<commit_after># Copyright (C) 2010-2013 Cuckoo Sandbox Developers.
# This file is part of Cuckoo Sandbox - http://www.cuckoosandbox.org
# See the file 'docs/LICENSE' for copying permission.

import os
import time
import shutil
import logging
import Queue
from threading import Thread, Lock

from lib.cuckoo.common.constants import CUCKOO_ROOT
from lib.cuckoo.common.exceptions import CuckooMachineError, CuckooGuestError, CuckooOperationalError, CuckooCriticalError
from lib.cuckoo.common.objects import File
from lib.cuckoo.common.utils import create_folder
from lib.cuckoo.common.config import Config
from lib.cuckoo.core.database import Database, TASK_COMPLETED, TASK_REPORTED
from lib.cuckoo.core.guest import GuestManager
from lib.cuckoo.core.resultserver import Resultserver
from lib.cuckoo.core.plugins import list_plugins, RunAuxiliary, RunProcessing, RunSignatures, RunReporting

log = logging.getLogger(__name__)

machinery = None
machine_lock = Lock()

total_analysis_count = 0
active_analysis_count = 0

class AnalysisManager(Thread):
    """Analysis Manager.

    This class handles the full analysis process for a given task. It takes
    care of selecting the analysis machine, preparing the configuration and
    interacting with the guest agent and analyzer components to launch and
    complete the analysis and store, process and report its results.
    """

    def __init__(self, task, error_queue):
        """@param task: task object containing the details for the analysis."""
        Thread.__init__(self)
        Thread.daemon = True

        self.task = task
        self.errors = error_queue
        self.cfg = Config()
        self.storage = ""
        self.binary = ""
        self.machine = None

    def init_storage(self):
        """Initialize analysis storage folder."""
        self.storage = os.path.join(CUCKOO_ROOT,
                                    "storage",
                                    "analyses",
                                    str(self.task.id))

        # If the analysis storage folder already exists, we need to abort the
        # analysis or previous results will be overwritten and lost.
        if os.path.exists(self.storage):
            log.error("Analysis results folder already exists at path \"%s\","
                      " analysis aborted", self.storage)
            return False

        # If we're not able to create the analysis storage folder, we have to
        # abort the analysis.
        try:
            create_folder(folder=self.storage)
        except CuckooOperationalError:
            log.error("Unable to create analysis folder %s", self.storage)
            return False

        return True

    def store_file(self):
        """Store a copy of the file being analyzed."""
        if not os.path.exists(self.task.target):
            log.error("The file to analyze does not exist at path \"%s\", "
                      "analysis aborted", self.task.target)
            return False

        sha256 = File(self.task.target).get_sha256()
        self.binary = os.path.join(CUCKOO_ROOT, "storage", "binaries", sha256)

        if os.path.exists(self.binary):
            log.info("File already exists at \"%s\"", self.binary)
        else:
            # TODO: do we really need to abort the analysis in case we are not
            # able to store a copy of the file?
            try:
                shutil.copy(self.task.target, self.binary)
            except (IOError, shutil.Error) as e:
                log.error("Unable to store file from \"%s\" to \"%s\", "
                          "analysis aborted", self.task.target, self.binary)
                return False

        try:
            new_binary_path = os.path.join(self.storage, "binary")

            if hasattr(os, "symlink"):
                os.symlink(self.binary, new_binary_path)
            else:
                shutil.copy(self.binary, new_binary_path)
        except (AttributeError, OSError) as e:
            log.error("Unable to create symlink/copy from \"%s\" to \"%s\"", self.binary, self.storage)

        return True

    def acquire_machine(self):
        """Acquire an analysis machine from the pool of available ones."""
        machine = None

        # Start a loop to acquire the a machine to run the analysis on.
        while True:
            machine_lock.acquire()
            # If the user specified a specific machine ID, a platform to be
            # used or machine tags acquire the machine accordingly.
            try:
                machine = machinery.acquire(machine_id=self.task.machine,
                                            platform=self.task.platform,
                                            tags=self.task.tags)
            finally:
                machine_lock.release()

            # If no machine is available at this moment, wait for one second
            # and try again.
            if not machine:
                log.debug("Task #%d: no machine available yet", self.task.id)
                time.sleep(1)
            else:
                log.info("Task #%d: acquired machine %s (label=%s)", self.task.id, machine.name, machine.label)
                break

        self.machine = machine

    def build_options(self):
        """Generate analysis options.
        @return: options dict.
        """
        options = {}

        options["id"] = self.task.id
        options["ip"] = self.machine.resultserver_ip
        options["port"] = self.machine.resultserver_port
        options["category"] = self.task.category
        options["target"] = self.task.target
        options["package"] = self.task.package
        options["options"] = self.task.options
        options["enforce_timeout"] = self.task.enforce_timeout
        options["clock"] = self.task.clock

        if not self.task.timeout or self.task.timeout == 0:
            options["timeout"] = self.cfg.timeouts.default
        else:
            options["timeout"] = self.task.timeout

        if self.task.category == "file":
            options["file_name"] = File(self.task.target).get_name()
            options["file_type"] = File(self.task.target).get_type()

        return options

    def launch_analysis(self):
        """Start analysis."""
        succeeded = False

        log.info("Starting analysis of %s \"%s\" (task=%d)", self.task.category.upper(), self.task.target, self.task.id)

        # Initialize the the analysis folders.
        if not self.init_storage():
            return False

        if self.task.category == "file":
            # Store a copy of the original file.
            if not self.store_file():
                return False

        # Acquire analysis machine.
        try:
            self.acquire_machine()
        except CuckooOperationalError as e:
            log.error("Cannot acquire machine: {0}".format(e))
            return False

        # Generate the analysis configuration file.
        options = self.build_options()

        # At this point we can tell the Resultserver about it.
        try:
            Resultserver().add_task(self.task, self.machine)
        except Exception as e:
            machinery.release(self.machine.label)
            self.errors.put(e)

        aux = RunAuxiliary(task=self.task, machine=self.machine)
        aux.start()

        try:
            # Mark the selected analysis machine in the database as started.
            guest_log = Database().guest_start(self.task.id,
                                               self.machine.name,
                                               self.machine.label,
                                               machinery.__class__.__name__)
            # Start the machine.
            machinery.start(self.machine.label)
        except CuckooMachineError as e:
            log.error(str(e), extra={"task_id": self.task.id})

            # Stop Auxiliary modules.
            aux.stop()

            return False
        else:
            try:
                # Initialize the guest manager.
                guest = GuestManager(self.machine.name, self.machine.ip, self.machine.platform)
                # Start the analysis.
                guest.start_analysis(options)
            except CuckooGuestError as e:
                log.error(str(e), extra={"task_id": self.task.id})

                # Stop Auxiliary modules.
                aux.stop()

                return False
            else:
                # Wait for analysis completion.
                try:
                    guest.wait_for_completion()
                    succeeded = True
                except CuckooGuestError as e:
                    log.error(str(e), extra={"task_id": self.task.id})
                    succeeded = False

        finally:
            # Stop Auxiliary modules.
            aux.stop()

            # Take a memory dump of the machine before shutting it off.
            if self.cfg.cuckoo.memory_dump or self.task.memory:
                try:
                    machinery.dump_memory(self.machine.label,
                                          os.path.join(self.storage, "memory.dmp"))
                except NotImplementedError:
                    log.error("The memory dump functionality is not available "
                              "for current machine manager")
                except CuckooMachineError as e:
                    log.error(e)

            try:
                # Stop the analysis machine.
                machinery.stop(self.machine.label)
            except CuckooMachineError as e:
                log.warning("Unable to stop machine %s: %s", self.machine.label, e)

            # Market the machine in the database as stopped.
            Database().guest_stop(guest_log)

            try:
                # Release the analysis machine.
                machinery.release(self.machine.label)
            except CuckooMachineError as e:
                log.error("Unable to release machine %s, reason %s. "
                          "You might need to restore it manually", self.machine.label, e)

            # after all this, we can make the Resultserver forget about it
            Resultserver().del_task(self.task, self.machine)

        return succeeded

    def process_results(self):
        """Process the analysis results and generate the enabled reports."""
        results = RunProcessing(task_id=self.task.id).run()
        RunSignatures(results=results).run()
        RunReporting(task_id=self.task.id, results=results).run()

        for proc in results["behavior"]["processes"]:
            log.debug("ParseProcessLog instance for %d (%s) parsed its log %d times.",
                proc["process_id"], proc["process_name"], proc["calls"].parsecount)

        # If the target is a file and the user enabled the option,
        # delete the original copy.
        if self.task.category == "file" and self.cfg.cuckoo.delete_original:
            try:
                os.remove(self.task.target)
            except OSError as e:
                log.error("Unable to delete original file at path \"%s\": %s", self.task.target, e)

        log.info("Task #%d: reports generation completed (path=%s)", self.task.id, self.storage)

        return True

    def run(self):
        """Run manager thread."""
        global active_analysis_count
        active_analysis_count += 1
        success = self.launch_analysis()
        Database().set_status(self.task.id, TASK_COMPLETED)

        log.debug("Released database task #%d with status %s", self.task.id, success)

        self.process_results()
        Database().set_status(self.task.id, TASK_REPORTED)

        log.info("Task #%d: analysis procedure completed", self.task.id)
        active_analysis_count -= 1

class Scheduler:
    """Tasks Scheduler.

    This class is responsible for the main execution loop of the tool. It
    prepares the analysis machines and keep waiting and loading for new
    analysis tasks.
    Whenever a new task is available, it launches AnalysisManager which will
    take care of running the full analysis process and operating with the
    assigned analysis machine.
    """

    def __init__(self):
        self.running = True
        self.cfg = Config()
        self.db = Database()

    def initialize(self):
        """Initialize the machine manager."""
        global machinery

        machinery_name = self.cfg.cuckoo.machine_manager

        log.info("Using \"%s\" machine manager", machinery_name)

        # Get registered class name. Only one machine manager is imported,
        # therefore there should be only one class in the list.
        plugin = list_plugins("machinery")[0]
        # Initialize the machine manager.
        machinery = plugin()

        # Find its configuration file.
        conf = os.path.join(CUCKOO_ROOT, "conf", "%s.conf" % machinery_name)

        if not os.path.exists(conf):
            raise CuckooCriticalError("The configuration file for machine "
                                      "manager \"{0}\" does not exist at path: "
                                      "{1}".format(machinery_name, conf))

        # Provide a dictionary with the configuration options to the
        # machine manager instance.
        machinery.set_options(Config(conf))
        # Initialize the machine manager.
        machinery.initialize(machinery_name)

        # At this point all the available machines should have been identified
        # and added to the list. If none were found, Cuckoo needs to abort the
        # execution.
        if len(machinery.machines()) == 0:
            raise CuckooCriticalError("No machines available")
        else:
            log.info("Loaded %s machine/s", len(machinery.machines()))

    def stop(self):
        """Stop scheduler."""
        self.running = False
        # Shutdown machine manager (used to kill machines that still alive).
        machinery.shutdown()

    def start(self):
        """Start scheduler."""
        global total_analysis_count
        self.initialize()

        log.info("Waiting for analysis tasks...")

        # Message queue with threads to transmit exceptions (used as IPC).
        errors = Queue.Queue()

        maxcount = self.cfg.cuckoo.max_analysis_count

        # This loop runs forever.
        while self.running:
            time.sleep(1)

            # If no machines are available, it's pointless to fetch for
            # pending tasks. Loop over.
            if machinery.availables() == 0:
                continue

            if total_analysis_count >= maxcount:
                if active_analysis_count <= 0:
                    self.stop()
            else:
                # Fetch a pending analysis task.
                task = self.db.fetch()

                if task:
                    log.debug("Processing task #%s", task.id)
                    total_analysis_count += 1

                    # Initialize the analysis manager.
                    analysis = AnalysisManager(task, errors)
                    # Start.
                    analysis.start()

            # Deal with errors.
            try:
                error = errors.get(block=False)
            except Queue.Empty:
                pass
            else:
                raise error
<|endoftext|>


<commit_before>import os
import os.path as op
import pandas as pd
# from collections import Counter

from bcbio.utils import rbind, file_exists
from bcbio.provenance import do
from bcbio.distributed.transaction import file_transaction
from bcbio.log import logger

class cov_class:

    def __init__(self, size, name, sample):
        self.size = size
        self.name = name
        self.sample = sample
        self.cov = {'10': 0, '25': 0, '50': 0}

    def save(self, cov, pt):
        if cov > 10:
            self.cov['10'] += pt
        if cov > 25:
            self.cov['25'] += pt
        if cov > 50:
            self.cov['50'] += pt

    def dataframe(self, out_file):
        # names = ["region", "size", "sample", "10", "25", "50"]
        df = pd.DataFrame(self.cov, index=["1"])
        df["region"] = self.name
        df["size"] = self.size
        df["sample"] = self.sample
        df.to_csv(out_file, mode='a', index=False, sep="\t")
        #return df

def _get_exome_coverage_stats(fn, sample, out_file):
    tmp_region = ""
    stats = ""
    with open(fn) as in_handle:
        for line in in_handle:
            if line.startswith("all"):
                continue
            cols = line.strip().split()
            cur_region = "_".join(cols[0:2])
            if cur_region != tmp_region:
                if tmp_region != "":
                    stats.dataframe(out_file)
                stats = cov_class(cols[-2], cur_region, sample)
            stats.save(int(cols[-4]), float(cols[-1]))
            tmp_region = cur_region
        stats.dataframe(out_file)

def _calc_total_exome_coverage(data, args):
    in_bam = data['bam']
    out_dir = args.out
    bed_file = args.region
    sample = op.splitext(os.path.basename(in_bam))[0]
    cov_file = op.join(out_dir, sample + ".dat")
    parse_file = op.join(out_dir, sample + "_cov.tsv")
    if not file_exists(cov_file):
        with file_transaction(cov_file) as cov_tx:
            cmd = ("bedtools coverage -abam {in_bam} -b {bed_file} -hist > {cov_tx}")
            do.run(cmd.format(**locals()), "exome coverage for %s" % in_bam)
    if not file_exists(parse_file):
        with file_transaction(parse_file) as out_tx:
            logger.info('parsing coverage: %s' % sample)
            _get_exome_coverage_stats(cov_file, sample, out_tx)
    return cov_file

<commit_msg>update bedtool syntax<commit_after>import os
import os.path as op
import pandas as pd
# from collections import Counter

from bcbio.utils import rbind, file_exists
from bcbio.provenance import do
from bcbio.distributed.transaction import file_transaction
from bcbio.log import logger

class cov_class:

    def __init__(self, size, name, sample):
        self.size = size
        self.name = name
        self.sample = sample
        self.cov = {'10': 0, '25': 0, '50': 0}

    def save(self, cov, pt):
        if cov > 10:
            self.cov['10'] += pt
        if cov > 25:
            self.cov['25'] += pt
        if cov > 50:
            self.cov['50'] += pt

    def dataframe(self, out_file):
        # names = ["region", "size", "sample", "10", "25", "50"]
        df = pd.DataFrame(self.cov, index=["1"])
        df["region"] = self.name
        df["size"] = self.size
        df["sample"] = self.sample
        df.to_csv(out_file, mode='a', header=False, index=False, sep="\t")
        #return df

def _get_exome_coverage_stats(fn, sample, out_file):
    tmp_region = ""
    stats = ""
    with open(fn) as in_handle:
        for line in in_handle:
            if line.startswith("all"):
                continue
            cols = line.strip().split()
            cur_region = "_".join(cols[0:2])
            if cur_region != tmp_region:
                if tmp_region != "":
                    stats.dataframe(out_file)
                stats = cov_class(cols[-2], cur_region, sample)
            stats.save(int(cols[-4]), float(cols[-1]))
            tmp_region = cur_region
        stats.dataframe(out_file)

def _calc_total_exome_coverage(data, args):
    in_bam = data['bam']
    out_dir = args.out
    bed_file = args.region
    sample = op.splitext(os.path.basename(in_bam))[0]
    cov_file = op.join(out_dir, sample + ".dat")
    parse_file = op.join(out_dir, sample + "_cov.tsv")
    if not file_exists(cov_file):
        with file_transaction(cov_file) as cov_tx:
            cmd = ("bedtools coverage -b {in_bam} -a {bed_file} -hist > {cov_tx}")
            do.run(cmd.format(**locals()), "exome coverage for %s" % in_bam)
    if not file_exists(parse_file):
        with file_transaction(parse_file) as out_tx:
            with open(out_tx, 'w') as out_handle:
                print >>out_handle, "q10\tq25\tq50\tregion\tsize\tsample"
            logger.info('parsing coverage: %s' % sample)
            _get_exome_coverage_stats(cov_file, sample, out_tx)
    return cov_file

<|endoftext|>


<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Python helper for loading hadoop ops and kernels."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from tensorflow.contrib.util import loader
from tensorflow.python.platform import resource_loader

_dataset_ops = loader.load_op_library(
    resource_loader.get_path_to_datafile("../../_dataset_ops.so"))
<commit_msg>Remove tensorflow_io/hadoop/python/ops/hadoop_op_loader.py<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2015-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test BIP66 (DER SIG).

Test the DERSIG soft-fork activation on regtest.
"""

from test_framework.blocktools import (
    DERSIG_HEIGHT,
    create_block,
    create_coinbase,
)
from test_framework.messages import msg_block
from test_framework.p2p import P2PInterface
from test_framework.script import CScript
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
    assert_equal,
)
from test_framework.wallet import (
    MiniWallet,
    MiniWalletMode,
)

# A canonical signature consists of:
# <30> <total len> <02> <len R> <R> <02> <len S> <S> <hashtype>
def unDERify(tx):
    """
    Make the signature in vin 0 of a tx non-DER-compliant,
    by adding padding after the S-value.
    """
    scriptSig = CScript(tx.vin[0].scriptSig)
    newscript = []
    for i in scriptSig:
        if (len(newscript) == 0):
            newscript.append(i[0:-1] + b'\0' + i[-1:])
        else:
            newscript.append(i)
    tx.vin[0].scriptSig = CScript(newscript)

class BIP66Test(BitcoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 1
        self.extra_args = [[
            '-whitelist=noban@127.0.0.1',
            '-par=1',  # Use only one script thread to get the exact log msg for testing
        ]]
        self.setup_clean_chain = True
        self.rpc_timeout = 240

    def create_tx(self, input_txid):
        utxo_to_spend = self.miniwallet.get_utxo(txid=input_txid, mark_as_spent=False)
        return self.miniwallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_to_spend)['tx']

    def test_dersig_info(self, *, is_active):
        assert_equal(self.nodes[0].getblockchaininfo()['softforks']['bip66'],
            {
                "active": is_active,
                "height": DERSIG_HEIGHT,
                "type": "buried",
            },
        )

    def run_test(self):
        peer = self.nodes[0].add_p2p_connection(P2PInterface())
        self.miniwallet = MiniWallet(self.nodes[0], mode=MiniWalletMode.RAW_P2PK)

        self.test_dersig_info(is_active=False)

        self.log.info("Mining %d blocks", DERSIG_HEIGHT - 2)
        self.coinbase_txids = [self.nodes[0].getblock(b)['tx'][0] for b in self.miniwallet.generate(DERSIG_HEIGHT - 2)]

        self.log.info("Test that a transaction with non-DER signature can still appear in a block")

        spendtx = self.create_tx(self.coinbase_txids[0])
        unDERify(spendtx)
        spendtx.rehash()

        tip = self.nodes[0].getbestblockhash()
        block_time = self.nodes[0].getblockheader(tip)['mediantime'] + 1
        block = create_block(int(tip, 16), create_coinbase(DERSIG_HEIGHT - 1), block_time)
        block.nVersion = 2
        block.vtx.append(spendtx)
        block.hashMerkleRoot = block.calc_merkle_root()
        block.rehash()
        block.solve()

        assert_equal(self.nodes[0].getblockcount(), DERSIG_HEIGHT - 2)
        self.test_dersig_info(is_active=False)  # Not active as of current tip and next block does not need to obey rules
        peer.send_and_ping(msg_block(block))
        assert_equal(self.nodes[0].getblockcount(), DERSIG_HEIGHT - 1)
        self.test_dersig_info(is_active=True)  # Not active as of current tip, but next block must obey rules
        assert_equal(self.nodes[0].getbestblockhash(), block.hash)

        self.log.info("Test that blocks must now be at least version 3")
        tip = block.sha256
        block_time += 1
        block = create_block(tip, create_coinbase(DERSIG_HEIGHT), block_time)
        block.nVersion = 2
        block.rehash()
        block.solve()

        with self.nodes[0].assert_debug_log(expected_msgs=['{}, bad-version(0x00000002)'.format(block.hash)]):
            peer.send_and_ping(msg_block(block))
            assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip)
            peer.sync_with_ping()

        self.log.info("Test that transactions with non-DER signatures cannot appear in a block")
        block.nVersion = 3

        spendtx = self.create_tx(self.coinbase_txids[1])
        unDERify(spendtx)
        spendtx.rehash()

        # First we show that this tx is valid except for DERSIG by getting it
        # rejected from the mempool for exactly that reason.
        assert_equal(
            [{
                'txid': spendtx.hash,
                'wtxid': spendtx.getwtxid(),
                'allowed': False,
                'reject-reason': 'non-mandatory-script-verify-flag (Non-canonical DER signature)',
            }],
            self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0),
        )

        # Now we verify that a block with this transaction is also invalid.
        block.vtx.append(spendtx)
        block.hashMerkleRoot = block.calc_merkle_root()
        block.rehash()
        block.solve()

        with self.nodes[0].assert_debug_log(expected_msgs=['CheckInputScripts on {} failed with non-mandatory-script-verify-flag (Non-canonical DER signature)'.format(block.vtx[-1].hash)]):
            peer.send_and_ping(msg_block(block))
            assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip)
            peer.sync_with_ping()

        self.log.info("Test that a version 3 block with a DERSIG-compliant transaction is accepted")
        block.vtx[1] = self.create_tx(self.coinbase_txids[1])
        block.hashMerkleRoot = block.calc_merkle_root()
        block.rehash()
        block.solve()

        self.test_dersig_info(is_active=True)  # Not active as of current tip, but next block must obey rules
        peer.send_and_ping(msg_block(block))
        self.test_dersig_info(is_active=True)  # Active as of current tip
        assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)

if __name__ == '__main__':
    BIP66Test().main()
<commit_msg>test: use f-strings in feature_dersig.py<commit_after>#!/usr/bin/env python3
# Copyright (c) 2015-2020 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Test BIP66 (DER SIG).

Test the DERSIG soft-fork activation on regtest.
"""

from test_framework.blocktools import (
    DERSIG_HEIGHT,
    create_block,
    create_coinbase,
)
from test_framework.messages import msg_block
from test_framework.p2p import P2PInterface
from test_framework.script import CScript
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import (
    assert_equal,
)
from test_framework.wallet import (
    MiniWallet,
    MiniWalletMode,
)

# A canonical signature consists of:
# <30> <total len> <02> <len R> <R> <02> <len S> <S> <hashtype>
def unDERify(tx):
    """
    Make the signature in vin 0 of a tx non-DER-compliant,
    by adding padding after the S-value.
    """
    scriptSig = CScript(tx.vin[0].scriptSig)
    newscript = []
    for i in scriptSig:
        if (len(newscript) == 0):
            newscript.append(i[0:-1] + b'\0' + i[-1:])
        else:
            newscript.append(i)
    tx.vin[0].scriptSig = CScript(newscript)

class BIP66Test(BitcoinTestFramework):
    def set_test_params(self):
        self.num_nodes = 1
        self.extra_args = [[
            '-whitelist=noban@127.0.0.1',
            '-par=1',  # Use only one script thread to get the exact log msg for testing
        ]]
        self.setup_clean_chain = True
        self.rpc_timeout = 240

    def create_tx(self, input_txid):
        utxo_to_spend = self.miniwallet.get_utxo(txid=input_txid, mark_as_spent=False)
        return self.miniwallet.create_self_transfer(from_node=self.nodes[0], utxo_to_spend=utxo_to_spend)['tx']

    def test_dersig_info(self, *, is_active):
        assert_equal(self.nodes[0].getblockchaininfo()['softforks']['bip66'],
            {
                "active": is_active,
                "height": DERSIG_HEIGHT,
                "type": "buried",
            },
        )

    def run_test(self):
        peer = self.nodes[0].add_p2p_connection(P2PInterface())
        self.miniwallet = MiniWallet(self.nodes[0], mode=MiniWalletMode.RAW_P2PK)

        self.test_dersig_info(is_active=False)

        self.log.info("Mining %d blocks", DERSIG_HEIGHT - 2)
        self.coinbase_txids = [self.nodes[0].getblock(b)['tx'][0] for b in self.miniwallet.generate(DERSIG_HEIGHT - 2)]

        self.log.info("Test that a transaction with non-DER signature can still appear in a block")

        spendtx = self.create_tx(self.coinbase_txids[0])
        unDERify(spendtx)
        spendtx.rehash()

        tip = self.nodes[0].getbestblockhash()
        block_time = self.nodes[0].getblockheader(tip)['mediantime'] + 1
        block = create_block(int(tip, 16), create_coinbase(DERSIG_HEIGHT - 1), block_time)
        block.nVersion = 2
        block.vtx.append(spendtx)
        block.hashMerkleRoot = block.calc_merkle_root()
        block.rehash()
        block.solve()

        assert_equal(self.nodes[0].getblockcount(), DERSIG_HEIGHT - 2)
        self.test_dersig_info(is_active=False)  # Not active as of current tip and next block does not need to obey rules
        peer.send_and_ping(msg_block(block))
        assert_equal(self.nodes[0].getblockcount(), DERSIG_HEIGHT - 1)
        self.test_dersig_info(is_active=True)  # Not active as of current tip, but next block must obey rules
        assert_equal(self.nodes[0].getbestblockhash(), block.hash)

        self.log.info("Test that blocks must now be at least version 3")
        tip = block.sha256
        block_time += 1
        block = create_block(tip, create_coinbase(DERSIG_HEIGHT), block_time)
        block.nVersion = 2
        block.rehash()
        block.solve()

        with self.nodes[0].assert_debug_log(expected_msgs=[f'{block.hash}, bad-version(0x00000002)']):
            peer.send_and_ping(msg_block(block))
            assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip)
            peer.sync_with_ping()

        self.log.info("Test that transactions with non-DER signatures cannot appear in a block")
        block.nVersion = 3

        spendtx = self.create_tx(self.coinbase_txids[1])
        unDERify(spendtx)
        spendtx.rehash()

        # First we show that this tx is valid except for DERSIG by getting it
        # rejected from the mempool for exactly that reason.
        assert_equal(
            [{
                'txid': spendtx.hash,
                'wtxid': spendtx.getwtxid(),
                'allowed': False,
                'reject-reason': 'non-mandatory-script-verify-flag (Non-canonical DER signature)',
            }],
            self.nodes[0].testmempoolaccept(rawtxs=[spendtx.serialize().hex()], maxfeerate=0),
        )

        # Now we verify that a block with this transaction is also invalid.
        block.vtx.append(spendtx)
        block.hashMerkleRoot = block.calc_merkle_root()
        block.rehash()
        block.solve()

        with self.nodes[0].assert_debug_log(expected_msgs=[f'CheckInputScripts on {block.vtx[-1].hash} failed with non-mandatory-script-verify-flag (Non-canonical DER signature)']):
            peer.send_and_ping(msg_block(block))
            assert_equal(int(self.nodes[0].getbestblockhash(), 16), tip)
            peer.sync_with_ping()

        self.log.info("Test that a version 3 block with a DERSIG-compliant transaction is accepted")
        block.vtx[1] = self.create_tx(self.coinbase_txids[1])
        block.hashMerkleRoot = block.calc_merkle_root()
        block.rehash()
        block.solve()

        self.test_dersig_info(is_active=True)  # Not active as of current tip, but next block must obey rules
        peer.send_and_ping(msg_block(block))
        self.test_dersig_info(is_active=True)  # Active as of current tip
        assert_equal(int(self.nodes[0].getbestblockhash(), 16), block.sha256)

if __name__ == '__main__':
    BIP66Test().main()
<|endoftext|>


<commit_before>from __future__ import division
import logging
from libs.Adafruit_PWM_Servo_Driver import PWM
from numpy import interp, clip

class DriveTrain():
    """Instantiate a 2WD drivetrain, utilising 2x ESCs,
    controlled using a 2 axis (throttle, steering)
    system + skittle accessories"""
    def __init__(
        self,
        pwm_i2c=0x40,
        pwm_freq=50,
        left_channel=0,
        right_channel=1,
        aux_channel1=4,
        aux_channel2=5,
        aux_channel3=6,
        aux_channel4=7,
        debug=False
    ):
        # Main set of motor controller ranges
        self.servo_min = 900
        #self.servo_mid = 1555
        self.servo_mid = 1650
        self.servo_max = 2300

        # Full speed range
        self.servo_full_min = 900
        self.servo_full_max = 2300
        # Low speed range is 1/4 of full speed
        speed_divisor = 2
        self.servo_low_min = int(self.servo_mid - (self.servo_mid-self.servo_full_min) / speed_divisor)
        self.servo_low_max = int(self.servo_mid + (self.servo_full_max-self.servo_mid) / speed_divisor)

        # Skittle launcher servos
        self.skittle_left_servo_closed = 1400  # 1600
        self.skittle_left_servo_open = 1200  # 1400
        self.skittle_right_servo_closed = 1400  # 1600
        self.skittle_right_servo_open = 1200  # 1400
        self.skittle_left_servo_init = 1300
        self.skittle_right_servo_init = 1300

        # Skittle launcher motors
        self.skittle_left_motor_stopped = 2000
        self.skittle_left_motor_full_speed = 1000
        self.skittle_right_motor_stopped = 2000
        self.skittle_right_motor_full_speed = 1000

        # Proximity probe servo limites
        self.proximity_servo_min = 1000
        self.proximity_servo_max = 2000

        # Proximity probe fan limits
        self.proximity_blower_esc_stopped = 1000
        self.proximity_blower_esc_max = 2000

        self.channels = {
            'left': left_channel,
            'right': right_channel,
            'blower': aux_channel1,
            'probe_servo': aux_channel2,
            'skittle_left_servo': aux_channel1,
            'skittle_right_servo': aux_channel2,
            'skittle_left_motor': aux_channel3,
            'skittle_right_motor': aux_channel4
        }

        self.pwm = PWM(pwm_i2c, debug=debug)
        self.pwm.setPWMFreq(pwm_freq)
        # Flag set to True when motors are allowed to move
        self.drive_enabled = False
        self.disable_drive()

    def set_servo_pulse(self, channel, pulse):
        """Send a raw servo pulse length to a specific speed controller
        channel"""
        # Only send servo pulses if drive is enabled
        if self.drive_enabled:
            # 1,000,000 us per second
            pulseLength = 1000000
            #  60 Hz
            pulseLength /= 50
            # logging.debug("%d us per period" % pulseLength)
            # 12 bits of resolution
            pulseLength /= 4096
            # logging.debug("%d us per bit" % pulseLength)
            # pulse *= 1000
            pulse /= pulseLength
            logging.debug("pulse {0} - channel {1}".format(int(pulse), channel))
            self.pwm.setPWM(channel, 0, int(pulse))

    def enable_drive(self):
        """Allow motors to be used"""
        self.drive_enabled = True

    def disable_drive(self):
        """Disable motors so they cant be used"""
        self.set_neutral()
        self.drive_enabled = False

    def set_neutral(self):
        """Send the neutral servo position to both motor controllers"""
        self.set_servo_pulse(self.channels['left'], self.servo_mid)
        self.set_servo_pulse(self.channels['right'], self.servo_mid)

    def set_full_speed(self):
        """Set servo range to FULL extents"""
        self.servo_min = self.servo_full_min
        self.servo_max = self.servo_full_max

    def set_low_speed(self):
        """Limit servo range extents"""
        self.servo_min = self.servo_low_min
        self.servo_max = self.servo_low_max

    # TODO - flesh out setters for raw pulse values (both channels)
    def mix_channels_and_assign(self, throttle, steering):
        """Take values for the throttle and steering channels in the range
        -1 to 1, convert into servo pulses, and then mix the channels and
        assign to the left/right motor controllers"""
        if not self.drive_enabled:
            return
        pulse_throttle = self._map_channel_value(throttle)
        pulse_steering = self._map_channel_value(steering)
        output_pulse_left = clip(
            #(-pulse_throttle + pulse_steering) / 2 + self.servo_mid,
            (pulse_throttle + pulse_steering) / 2,
            self.servo_min,
            self.servo_max
        )
        output_pulse_right = clip(
            #(pulse_throttle + pulse_steering) / 2,
            (pulse_throttle - pulse_steering) / 2 + self.servo_mid,
            self.servo_min,
            self.servo_max
        )

        logging.info(
            "output pulse left: {0} output pulse right : {1}".format(
                output_pulse_left, output_pulse_right
            )
        )

        # Set the servo pulses for left and right channels
        self.set_servo_pulse(self.channels['left'], output_pulse_left)
        self.set_servo_pulse(self.channels['right'], output_pulse_right)

    def _map_channel_value(self, value):
        """Map the supplied value from the range -1 to 1 to a corresponding
        value within the range servo_min to servo_max"""
        return int(
            interp(
                value,
                [-1, 1],
                [self.servo_min, self.servo_max]
            )
        )

    def set_skittle_arms_closed(self):
        """Send the back servo position to both skittle servos"""
        self.set_servo_pulse(
            self.channels['skittle_left_servo'],
            self.skittle_left_servo_closed
        )
        self.set_servo_pulse(
            self.channels['skittle_right_servo'],
            self.skittle_right_servo_closed
        )

    def set_skittle_arms_open(self):
        """Send the forward servo position to both skittle servos"""
        self.set_servo_pulse(
            self.channels['skittle_left_servo'],
            self.skittle_left_servo_open
        )
        self.set_servo_pulse(
            self.channels['skittle_right_servo'],
            self.skittle_right_servo_open
        )

    def set_skittle_motors_on(self):
        """Send the on servo position to both skittle escs"""
        self.set_servo_pulse(
            self.channels['skittle_left_motor'],
            self.skittle_left_motor_full_speed
        )
        self.set_servo_pulse(
            self.channels['skittle_right_motor'],
            self.skittle_right_motor_full_speed
        )

    def set_skittle_motors_off(self):
        """Send the stopped servo position to both skittle escs"""
        self.set_servo_pulse(
            self.channels['skittle_left_motor'],
            self.skittle_left_motor_stopped
        )
        self.set_servo_pulse(
            self.channels['skittle_right_motor'],
            self.skittle_right_motor_stopped
        )
<commit_msg>tweak right servo values<commit_after>from __future__ import division
import logging
from libs.Adafruit_PWM_Servo_Driver import PWM
from numpy import interp, clip

class DriveTrain():
    """Instantiate a 2WD drivetrain, utilising 2x ESCs,
    controlled using a 2 axis (throttle, steering)
    system + skittle accessories"""
    def __init__(
        self,
        pwm_i2c=0x40,
        pwm_freq=50,
        left_channel=0,
        right_channel=1,
        aux_channel1=4,
        aux_channel2=5,
        aux_channel3=6,
        aux_channel4=7,
        debug=False
    ):
        # Main set of motor controller ranges
        self.servo_min = 900
        #self.servo_mid = 1555
        self.servo_mid = 1650
        self.servo_max = 2300

        # Full speed range
        self.servo_full_min = 900
        self.servo_full_max = 2300
        # Low speed range is 1/4 of full speed
        speed_divisor = 2
        self.servo_low_min = int(self.servo_mid - (self.servo_mid-self.servo_full_min) / speed_divisor)
        self.servo_low_max = int(self.servo_mid + (self.servo_full_max-self.servo_mid) / speed_divisor)

        # Skittle launcher servos
        self.skittle_left_servo_closed = 1400  # 1600
        self.skittle_left_servo_open = 1200  # 1400
        self.skittle_right_servo_closed = 1300
        self.skittle_right_servo_open = 1700
        self.skittle_left_servo_init = 1300
        self.skittle_right_servo_init = 1300

        # Skittle launcher motors
        self.skittle_left_motor_stopped = 2000
        self.skittle_left_motor_full_speed = 1000
        self.skittle_right_motor_stopped = 2000
        self.skittle_right_motor_full_speed = 1000

        # Proximity probe servo limites
        self.proximity_servo_min = 1000
        self.proximity_servo_max = 2000

        # Proximity probe fan limits
        self.proximity_blower_esc_stopped = 1000
        self.proximity_blower_esc_max = 2000

        self.channels = {
            'left': left_channel,
            'right': right_channel,
            'blower': aux_channel1,
            'probe_servo': aux_channel2,
            'skittle_left_servo': aux_channel1,
            'skittle_right_servo': aux_channel2,
            'skittle_left_motor': aux_channel3,
            'skittle_right_motor': aux_channel4
        }

        self.pwm = PWM(pwm_i2c, debug=debug)
        self.pwm.setPWMFreq(pwm_freq)
        # Flag set to True when motors are allowed to move
        self.drive_enabled = False
        self.disable_drive()

    def set_servo_pulse(self, channel, pulse):
        """Send a raw servo pulse length to a specific speed controller
        channel"""
        # Only send servo pulses if drive is enabled
        if self.drive_enabled:
            # 1,000,000 us per second
            pulseLength = 1000000
            #  60 Hz
            pulseLength /= 50
            # logging.debug("%d us per period" % pulseLength)
            # 12 bits of resolution
            pulseLength /= 4096
            # logging.debug("%d us per bit" % pulseLength)
            # pulse *= 1000
            pulse /= pulseLength
            logging.debug("pulse {0} - channel {1}".format(int(pulse), channel))
            self.pwm.setPWM(channel, 0, int(pulse))

    def enable_drive(self):
        """Allow motors to be used"""
        self.drive_enabled = True

    def disable_drive(self):
        """Disable motors so they cant be used"""
        self.set_neutral()
        self.drive_enabled = False

    def set_neutral(self):
        """Send the neutral servo position to both motor controllers"""
        self.set_servo_pulse(self.channels['left'], self.servo_mid)
        self.set_servo_pulse(self.channels['right'], self.servo_mid)

    def set_full_speed(self):
        """Set servo range to FULL extents"""
        self.servo_min = self.servo_full_min
        self.servo_max = self.servo_full_max

    def set_low_speed(self):
        """Limit servo range extents"""
        self.servo_min = self.servo_low_min
        self.servo_max = self.servo_low_max

    # TODO - flesh out setters for raw pulse values (both channels)
    def mix_channels_and_assign(self, throttle, steering):
        """Take values for the throttle and steering channels in the range
        -1 to 1, convert into servo pulses, and then mix the channels and
        assign to the left/right motor controllers"""
        if not self.drive_enabled:
            return
        pulse_throttle = self._map_channel_value(throttle)
        pulse_steering = self._map_channel_value(steering)
        output_pulse_left = clip(
            #(-pulse_throttle + pulse_steering) / 2 + self.servo_mid,
            (pulse_throttle + pulse_steering) / 2,
            self.servo_min,
            self.servo_max
        )
        output_pulse_right = clip(
            #(pulse_throttle + pulse_steering) / 2,
            (pulse_throttle - pulse_steering) / 2 + self.servo_mid,
            self.servo_min,
            self.servo_max
        )

        logging.info(
            "output pulse left: {0} output pulse right : {1}".format(
                output_pulse_left, output_pulse_right
            )
        )

        # Set the servo pulses for left and right channels
        self.set_servo_pulse(self.channels['left'], output_pulse_left)
        self.set_servo_pulse(self.channels['right'], output_pulse_right)

    def _map_channel_value(self, value):
        """Map the supplied value from the range -1 to 1 to a corresponding
        value within the range servo_min to servo_max"""
        return int(
            interp(
                value,
                [-1, 1],
                [self.servo_min, self.servo_max]
            )
        )

    def set_skittle_arms_closed(self):
        """Send the back servo position to both skittle servos"""
        self.set_servo_pulse(
            self.channels['skittle_left_servo'],
            self.skittle_left_servo_closed
        )
        self.set_servo_pulse(
            self.channels['skittle_right_servo'],
            self.skittle_right_servo_closed
        )

    def set_skittle_arms_open(self):
        """Send the forward servo position to both skittle servos"""
        self.set_servo_pulse(
            self.channels['skittle_left_servo'],
            self.skittle_left_servo_open
        )
        self.set_servo_pulse(
            self.channels['skittle_right_servo'],
            self.skittle_right_servo_open
        )

    def set_skittle_motors_on(self):
        """Send the on servo position to both skittle escs"""
        self.set_servo_pulse(
            self.channels['skittle_left_motor'],
            self.skittle_left_motor_full_speed
        )
        self.set_servo_pulse(
            self.channels['skittle_right_motor'],
            self.skittle_right_motor_full_speed
        )

    def set_skittle_motors_off(self):
        """Send the stopped servo position to both skittle escs"""
        self.set_servo_pulse(
            self.channels['skittle_left_motor'],
            self.skittle_left_motor_stopped
        )
        self.set_servo_pulse(
            self.channels['skittle_right_motor'],
            self.skittle_right_motor_stopped
        )
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
__author__="Josh Montague"
__license__="MIT License"

#
# This module defines a number of helper functions.
#

from datetime import datetime
import logging
import numpy as np
import os
import sys

def short_name(model):
    """Return a simplified name for this model. A bit brittle."""
    if hasattr(model, 'steps'):
        # pipeline
        name = '-'.join( [ pair[0] for pair in model.steps ] )
    elif hasattr(model, 'best_estimator_'):
        # gridsearchcv
        name = 'gscv_' + '-'.join( [x[0] for x in model.estimator.steps ])
    elif hasattr(model, 'base_estimator_'):
        # bagging
        name = 'bag_' + short_name(model.base_estimator)
    else:
        # for a single model, this will work
        name = model.__class__.__name__
    return name

def create_submission(predictions, sub_name, comment=None, team='DrJ'):
    """Include the specified array of image predictions in a
    properly-formatted submission file.
    """
    now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
    submission_name = '-'.join(sub_name.split())
    with open('submissions/{}_{}.submission'.format(
            now, submission_name), 'w') as f:

        f.write('#'*20 + ' Generated submission file\n')
        if comment is not None:
            f.write('# ' + comment + '\n')
        f.write('{}\n'.format(team))
        f.write('{}\n'.format(now))
        f.write('{}\n'.format(sub_name))
        for p in predictions:
            f.write('{}\n'.format(p))
    return True

def load_np_arrays():
    """Return numpy arrays for training dataset, training labels,
    and test dataset (in that order).
    """
    # nb: path assumes that we call this function from project root
    #   - may not be appropriate for importing into code in bin/ ?
    X_train = np.load(os.path.join('data', 'train-images.npy'))
    y_train = np.load(os.path.join('data', 'train-labels.npy'))
    X_test = np.load(os.path.join('data', 'test-images.npy'))

    return (X_train, y_train, X_test)

<commit_msg>Update utils.short_name for VotingClassifier<commit_after># -*- coding: UTF-8 -*-
__author__="Josh Montague"
__license__="MIT License"

#
# This module defines a number of helper functions.
#

from datetime import datetime
import logging
import numpy as np
import os
import sys

def short_name(model):
    """Return a simplified name for this model. A bit brittle."""
    # *TODO* make this method more general
    # for a single model, this will work
    name = model.__class__.__name__
    try:
        if hasattr(model, 'steps'):
            # pipeline
            name = '-'.join( [ pair[0] for pair in model.steps ] )
        elif hasattr(model, 'best_estimator_'):
            if hasattr(model.estimator, 'steps'):
                # gridsearchcv
                name = 'gscv_' + '-'.join( [x[0] for x in model.estimator.steps ])
            elif hasattr(model.estimator, 'estimators'):
                # votingclassifier
                name = 'gscv_vc_' + '-'.join( [x[0] for x in model.estimator.estimators ])
        elif hasattr(model, 'base_estimator_'):
            # bagging
            name = 'bag_' + short_name(model.base_estimator)
    except AttributeError, e:
        logging.info('utils.short_name() couldnt generate quality name')
        # for a single model, this will work
        name = model.__class__.__name__
        logging.info('falling back to generic name={}'.format(name))
    return name

def create_submission(predictions, sub_name, comment=None, team='DrJ'):
    """Include the specified array of image predictions in a
    properly-formatted submission file.
    """
    now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
    submission_name = '-'.join(sub_name.split())
    with open('submissions/{}_{}.submission'.format(
            now, submission_name), 'w') as f:

        f.write('#'*20 + ' Generated submission file\n')
        if comment is not None:
            f.write('# ' + comment + '\n')
        f.write('{}\n'.format(team))
        f.write('{}\n'.format(now))
        f.write('{}\n'.format(sub_name))
        for p in predictions:
            f.write('{}\n'.format(p))
    return True

def load_np_arrays():
    """Return numpy arrays for training dataset, training labels,
    and test dataset (in that order).
    """
    # nb: path assumes that we call this function from project root
    #   - may not be appropriate for importing into code in bin/ ?
    X_train = np.load(os.path.join('data', 'train-images.npy'))
    y_train = np.load(os.path.join('data', 'train-labels.npy'))
    X_test = np.load(os.path.join('data', 'test-images.npy'))

    return (X_train, y_train, X_test)

<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
import re
import logging
import urllib2
import json

TOKEN_DIR = '/tmp/tokens'
SHARE_DIR = '/var/www/virtualenv/bioshare/include/bioshare/media/files/'
ORIGINAL_COMMAND = None
USER = None
TEST = False
PERMISSIONS = {}
WRITE_PERMISSIONS = ['write_to_share','delete_share_files']
READ_PERMISSIONS = ['download_share_files']

import ConfigParser
config = ConfigParser.ConfigParser()
config.read('sshwrapper.config')

# r = re.compile('^hg -R (S%2B) serve --stdio$')
# match = re.search(r, os.environ['SSH_ORIGINAL_COMMAND'])
# if match:
#     repo_path = match.groups()[0]
#     if os.path.exists(repo_path):
#         os.execvp('hg', ['hg', '-R', repo_path, 'serve', '--stdio'])

# class WrapperException(Exception):
#     """Basically just the base Exception class"""

def get_token_data(token_file):
    import ConfigParser
    config = ConfigParser.ConfigParser()
    config.read(token_file)
    directory = config.get('details','directory')
    user = config.get('details','user')
    return {'directory':directory,'user':user}

def get_permissions(username,share):
    if not PERMISSIONS.has_key(share):
        url  = config.get('config','perm_api_url') % (share,username)
        response = urllib2.urlopen(url)
        response = json.load(response)
        PERMISSIONS[share]= response['permissions']
    return PERMISSIONS[share]

def can_write(username,share):
    perms = get_permissions(username,share)
    for perm in WRITE_PERMISSIONS:
        if perm not in perms:
            return False
    return True

def can_read(username,share):
    perms = get_permissions(username,share)
    for perm in READ_PERMISSIONS:
        if perm not in perms:
            return False
    return True

def transform_path(path):
    match = re.match('/(?P<token>[a-zA-Z0-9]{10})(?:/(?P<subpath>.*))', path)
    try:
        matches = match.groupdict()
        if not matches.has_key('token'):
            return None
        if matches.has_key('subpath'):
            if '..' in matches['subpath']:
                logger.info('Illegal subpath: %s' % matches['subpath'])
                return None
        token_file = join(TOKEN_DIR,match.group('token'))
        if isfile(token_file):
            data = get_token_data(token_file)
            if match.groupdict().has_key('subpath'):
                return join(data['directory'],match.group('subpath'))
            return data['directory']
        else:
            return None
    except:
        logger.info('Bad path: %s' % path)
        return None

def analyze_path(path):
#     print path
    match = re.match('/(?P<share>[a-zA-Z0-9]{15})(?:/(?P<subpath>.*))', path)
    try:
        matches = match.groupdict()
        if not matches.has_key('share'):
            raise Exception('analyze_path: Bad path: %s' % path)
        if matches.has_key('subpath'):
#             print matches['subpath']
            if '..' in matches['subpath']:
                raise Exception('Illegal subpath: %s' % matches['subpath'])
        if matches.has_key('subpath'):
            path = join(config.get('config','share_dir'), matches['share'], match.group('subpath'))
        else:
            path = join(config.get('config','share_dir'), matches['share'])
        return {'share':matches['share'],'path':path}
    except Exception, e:
        raise Exception('analyze_path: Bad path: %s' % path)
        logger.info('analyze_path exception: %s' % str(e))

def handle_rsync(parts):
    try:
        paths_data = [analyze_path(path) for path in parts[parts.index('.')+1:]]
        paths = [path_data['path'] for path_data in paths_data]
        shares = [path_data['share'] for path_data in paths_data]
        if '--sender' in parts:#server->client
            for share in shares:
                if not can_read(USER, share):
                    raise Exception('User %s cannot read from share %s' % (USER,share))
            command = ['rsync', '--server', '--sender', '-vrze.iLsf', '.'] + paths
        else:#client->server
            # --no-p --no-g --chmod=ugo=rwX  //destination default permissions
            for share in shares:
                if not can_write(USER, share):
                    raise Exception('User %s cannot write to share %s' % (USER,share))
            command = ['rsync', '--server', '-vrze.iLsf', '.'] + paths
#             command = parts[:4]+paths
        if TEST:
            logger.info('running rsync command: %s' % ', '.join(command))
            print command
        else:
            logger.info('running rsync command: %s' % ', '.join(command))
            os.execvp('rsync', command)
    except Exception, e:
        logger.info('handle_rsync exception: %s' % str(e))

def handle_ls(parts):
    path_data = analyze_path(parts[1])
    if path_data is not None:
        if can_read(USER, path_data['share']):
            command = ['ls', path_data['path']]
#             if TEST:
#                 print command
#             else:
            os.execvp('ls', command)
        else:
            logger.info( 'ls: Permission denied')
    else:
        logger.info( 'ls: Bad command')

def main():
    try:
        import shlex
        parts = shlex.split(ORIGINAL_COMMAND)
        logger.info('SSH_ORIGINAL_COMMAND: '+ORIGINAL_COMMAND)
        logger.info('SPLIT: '+', '.join(parts))
        if parts[0] == 'rsync':
            handle_rsync(parts)
        elif parts[0] == 'ls':
            handle_ls(parts)
        else:
            logger.info( 'Unsupported command: %s' % parts[0])
    #     os.execvp('ls', ['ls', '/var/www'])
    except Exception, e:
        logger.error('Bad or missing parameter "SSH_ORIGINAL_COMMAND"')

if __name__ == '__main__':
    #Should probably use argument parsing library, but trying to keep dependencies to a minumum

    if len(sys.argv)==2:
        USER = sys.argv[1]
    elif len(sys.argv)==3:
        USER = sys.argv[1]
        ORIGINAL_COMMAND = sys.argv[2] #for testing: ssh-wrapper.py username 'rsync /local/file remote:/TOKEN/subdir
        TEST = True
    if ORIGINAL_COMMAND is None:
        ORIGINAL_COMMAND = os.environ['SSH_ORIGINAL_COMMAND']
    logger = logging.getLogger('bioshare')
    hdlr = logging.FileHandler('/var/log/sshwrapper.log')
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.INFO)
    from os.path import join, isfile
    logger.info('user: %s' % USER)
    logger.info('command: %s' % ORIGINAL_COMMAND)
    main()
<commit_msg>write to stderr<commit_after>#!/usr/bin/env python
import os
import sys
import re
import logging
import urllib2
import json

TOKEN_DIR = '/tmp/tokens'
SHARE_DIR = '/var/www/virtualenv/bioshare/include/bioshare/media/files/'
ORIGINAL_COMMAND = None
USER = None
TEST = False
PERMISSIONS = {}
WRITE_PERMISSIONS = ['write_to_share','delete_share_files']
READ_PERMISSIONS = ['download_share_files']

import ConfigParser
config = ConfigParser.ConfigParser()
config.read('sshwrapper.config')

# r = re.compile('^hg -R (S%2B) serve --stdio$')
# match = re.search(r, os.environ['SSH_ORIGINAL_COMMAND'])
# if match:
#     repo_path = match.groups()[0]
#     if os.path.exists(repo_path):
#         os.execvp('hg', ['hg', '-R', repo_path, 'serve', '--stdio'])

# class WrapperException(Exception):
#     """Basically just the base Exception class"""

def get_token_data(token_file):
    import ConfigParser
    config = ConfigParser.ConfigParser()
    config.read(token_file)
    directory = config.get('details','directory')
    user = config.get('details','user')
    return {'directory':directory,'user':user}

def get_permissions(username,share):
    if not PERMISSIONS.has_key(share):
        url  = config.get('config','perm_api_url') % (share,username)
        response = urllib2.urlopen(url)
        response = json.load(response)
        PERMISSIONS[share]= response['permissions']
    return PERMISSIONS[share]

def can_write(username,share):
    perms = get_permissions(username,share)
    for perm in WRITE_PERMISSIONS:
        if perm not in perms:
            return False
    return True

def can_read(username,share):
    perms = get_permissions(username,share)
    for perm in READ_PERMISSIONS:
        if perm not in perms:
            return False
    return True

def transform_path(path):
    match = re.match('/(?P<token>[a-zA-Z0-9]{10})(?:/(?P<subpath>.*))', path)
    try:
        matches = match.groupdict()
        if not matches.has_key('token'):
            return None
        if matches.has_key('subpath'):
            if '..' in matches['subpath']:
                logger.info('Illegal subpath: %s' % matches['subpath'])
                return None
        token_file = join(TOKEN_DIR,match.group('token'))
        if isfile(token_file):
            data = get_token_data(token_file)
            if match.groupdict().has_key('subpath'):
                return join(data['directory'],match.group('subpath'))
            return data['directory']
        else:
            return None
    except:
        logger.info('Bad path: %s' % path)
        return None

def analyze_path(path):
#     print path
    match = re.match('/(?P<share>[a-zA-Z0-9]{15})(?:/(?P<subpath>.*))', path)
    try:
        matches = match.groupdict()
        if not matches.has_key('share'):
            raise Exception('analyze_path: Bad path: %s' % path)
        if matches.has_key('subpath'):
#             print matches['subpath']
            if '..' in matches['subpath']:
                raise Exception('Illegal subpath: %s' % matches['subpath'])
        if matches.has_key('subpath'):
            path = join(config.get('config','share_dir'), matches['share'], match.group('subpath'))
        else:
            path = join(config.get('config','share_dir'), matches['share'])
        return {'share':matches['share'],'path':path}
    except Exception, e:
        raise Exception('analyze_path: Bad path: %s' % path)

def handle_rsync(parts):
    try:
        paths_data = [analyze_path(path) for path in parts[parts.index('.')+1:]]
        paths = [path_data['path'] for path_data in paths_data]
        shares = [path_data['share'] for path_data in paths_data]
        if '--sender' in parts:#server->client
            for share in shares:
                if not can_read(USER, share):
                    raise Exception('User %s cannot read from share %s' % (USER,share))
            command = ['rsync', '--server', '--sender', '-vrze.iLsf', '.'] + paths
        else:#client->server
            # --no-p --no-g --chmod=ugo=rwX  //destination default permissions
            for share in shares:
                if not can_write(USER, share):
                    raise Exception('User %s cannot write to share %s' % (USER,share))
            command = ['rsync', '--server', '-vrze.iLsf', '.'] + paths
#             command = parts[:4]+paths
        if TEST:
            logger.info('running rsync command: %s' % ', '.join(command))
            print command
        else:
            logger.info('running rsync command: %s' % ', '.join(command))
            os.execvp('rsync', command)
    except Exception, e:
        print >> sys.stderr, 'handle_rsync exception: %s' % str(e)
        logger.info('handle_rsync exception: %s' % str(e))

def handle_ls(parts):
    path_data = analyze_path(parts[1])
    if path_data is not None:
        if can_read(USER, path_data['share']):
            command = ['ls', path_data['path']]
#             if TEST:
#                 print command
#             else:
            os.execvp('ls', command)
        else:
            logger.info( 'ls: Permission denied')
    else:
        logger.info( 'ls: Bad command')

def main():
    try:
        import shlex
        parts = shlex.split(ORIGINAL_COMMAND)
        logger.info('SSH_ORIGINAL_COMMAND: '+ORIGINAL_COMMAND)
        logger.info('SPLIT: '+', '.join(parts))
        if parts[0] == 'rsync':
            handle_rsync(parts)
        elif parts[0] == 'ls':
            handle_ls(parts)
        else:
            logger.info( 'Unsupported command: %s' % parts[0])
    #     os.execvp('ls', ['ls', '/var/www'])
    except Exception, e:
        logger.error('Bad or missing parameter "SSH_ORIGINAL_COMMAND"')

if __name__ == '__main__':
    #Should probably use argument parsing library, but trying to keep dependencies to a minumum

    if len(sys.argv)==2:
        USER = sys.argv[1]
    elif len(sys.argv)==3:
        USER = sys.argv[1]
        ORIGINAL_COMMAND = sys.argv[2] #for testing: ssh-wrapper.py username 'rsync /local/file remote:/TOKEN/subdir
        TEST = True
    if ORIGINAL_COMMAND is None:
        ORIGINAL_COMMAND = os.environ['SSH_ORIGINAL_COMMAND']
    logger = logging.getLogger('bioshare')
    hdlr = logging.FileHandler('/var/log/sshwrapper.log')
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    hdlr.setFormatter(formatter)
    logger.addHandler(hdlr)
    logger.setLevel(logging.INFO)
    from os.path import join, isfile
    logger.info('user: %s' % USER)
    logger.info('command: %s' % ORIGINAL_COMMAND)
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Small utility functions.
"""

import sys
import traceback
from random import betavariate
from random import choices
from math import floor

import logger

# various charsets
Alphabetics = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
Numbers = '0123456789'
Punctuation = """?,.!=/()'":;"""
AllUserChars = Alphabetics + Numbers + Punctuation

# all user chars in the 'Koch' order
Koch = """KMRSUAPTLOWI.NJE=F0Y,VG5/Q9ZH38B?427C1D6X():;!"'"""

# ensure Koch is same as AllUserChars
auc_sorted = ''.join(sorted(AllUserChars))
koch_sorted = ''.join(sorted(Koch))
if auc_sorted != koch_sorted:
    print('*'*80)
    print('* Error in utils.py, Koch and AllUserChars are different')
    print('*     len(Koch)=%d' % len(Koch))
    print('*     len(AllUserChars)=%d' % len(AllUserChars))
    print('*     auc_sorted= %s' % auc_sorted)
    print('*     koch_sorted=%s' % koch_sorted)
    print('*'*80)
    sys.exit(1)

# map chars to morse strings
Char2Morse = {
              '!': '-.-.--', '"': '.-..-.', '$': '...-..-', '&': '.-...',
              "'": '.----.', '(': '-.--.', ')': '-.--.-', ',': '--..--',
              '-': '-....-', '.': '.-.-.-', '/': '-..-.', ':': '---...',
              ';': '-.-.-.', '=': '-...-', '?': '..--..', '@': '.--.-.',
              '_': '..--.-', '+': '.-.-.',

              '0': '-----', '1': '.----', '2': '..---', '3': '...--',
              '4': '....-', '5': '.....', '6': '-....', '7': '--...',
              '8': '---..', '9': '----.',

              'A': '.-', 'B': '-...', 'C': '-.-.', 'D': '-..',
              'E': '.', 'F': '..-.', 'G': '--.', 'H': '....',
              'I': '..', 'J': '.---', 'K': '-.-', 'L': '.-..',
              'M': '--', 'N': '-.', 'O': '---', 'P': '.--.',
              'Q': '--.-', 'R': '.-.', 'S': '...', 'T': '-',
              'U': '..-', 'V': '...-', 'W': '.--', 'X': '-..-',
              'Y': '-.--', 'Z': '--..',

              ' ': ' '
             }

# invert above dict to create map of morse strings to chars
Morse2Char = {v:k for (k, v) in Char2Morse.items()}

# stylesheet code for PyQt5
#StyleCSS = ""
StyleCSS = """
/*css stylesheet file that contains all the style information*/

table, th, td {
    border: 1px solid black;
    border-collapse: collapse;
}

QGroupBox {
    border: 1px solid black;
    border-radius: 3px;
}

QGroupBox:title{
    subcontrol-origin: margin;
    subcontrol-position: top left;
    padding-left: 0px;
    padding-right: 5px;
    padding-top: -7px;
    border-top: none;
}
"""

log = logger.Log('debug.log', logger.Log.CRITICAL)

def str_trace(msg=None):
    """Get a traceback string.

    This is useful if we need at any point in code to find out how
    we got to that point.
    """

    result = []

    if msg:
        result.append(msg+'\n')

    result.extend(traceback.format_stack())

    return ''.join(result)

def log_trace(msg=None):
    """Log a traceback string."""

    log.debug(str_trace(msg))

def morse2display(morse):
    """Convert a string for a morse character to 'display' morse.

    We use this to display expected/received characters in the
    display as using ./- is not nice.  For example, '.-.' is not
    as nice as '• ━ •'
    """

    # unicode chars
    DOT = '•'    #2022
    DASH = '━'   #2501
    SIX_PER_EM_SPACE = ' '   #2005

    result = []
    for dotdash in morse:
        if dotdash == '.':
            result.append(DOT)
        elif dotdash in '-_':
            result.append(DASH)
        elif dotdash == ' ':
            result.append(' ')
        else:
            raise Exception("Unrecognized sign in '%s': %s" % (morse, dotdash))

    return SIX_PER_EM_SPACE.join(result)

def get_random_char(charset, stats):
    """Get a random char from the charset sequence.

    charset  a string of characters we are testing
    stats    a dictionary of stats: {'A': [T, F, T, ...], ...}

    Choose a character biased more towards the characters most in error.
    We do this by sorting the charset by error rate, then choosing from the
    front of the sorted list.

    A low number of samples biases character(s) to be more frequent.
    """

    # figure out the error rate of chars in the charset
    test_stats = {}
    for ch in charset:
        test_stats[ch] = stats[ch]
    ordered_charset = stats2errorrate(test_stats)

    bias = 0.5
    charset_len = len(charset)
    weights = [x+bias for x in range(charset_len)]
    sum_weights = sum(weights)
    normal_weights = [x/sum_weights for x in weights]

    return choices(ordered_charset, weights=normal_weights)[0]

def old_get_random_char(charset, stats):
    """Get a random char from the charset sequence.

    charset  a string of characters we are testing
    stats    a dictionary of stats: {'A': [T, F, T, ...], ...}

    Choose a character biased more towards the characters most in error.
    We do this by sorting the charset by error rate, then choosing from the
    front of the sorted list.
    """

    # figure out the error rate of chars in the charset
    test_stats = {}
    for ch in charset:
        test_stats[ch] = stats[ch]
    weighted_charset = stats2errorrate(test_stats)

    # choose randomly, but more likely the erroring char(s)
    beta = betavariate(1, 3)
    beta_len = beta * len(charset)
    rand_sample = floor(beta_len)
    result = weighted_charset[rand_sample]

    return result

def stats2errorrate(stats):
    """Convert stats data into an errorrate list.

    stats      dictionary holding statistics data

    The 'stats' data has the form {'A':[T,F,T], ...} where
    the list contains the last N results (True or False).

    The result is a list of characters sorted so the character with
    the highest error rate is forst, etc.
    """

    # working list of tuples (rate, char)
    temp = []

    # get a list of (rate, char)
    for (char, result_list) in stats.items():
        sample_size = len(result_list)
        rate = result_list.count(True) / sample_size
        if len(result_list) < 50:
            # if few results yet, test mostest, probably just added
            rate *= len(result_list)/50

        temp.append((rate, char))

    # sort by error rate, drop the rates
    temp2 = sorted(temp, key=lambda t: t[0], reverse=True)
    result = [ch for (_, ch) in temp2]

    return result

def char2morse(char):
    """Convert a character into a morse string."""

    return morse2display(Char2Morse[char.upper()])

def str2morse(string):
    """Convert a string into a morse string.

    Spaces are inserted between each character in morse.
    """

    return ' '.join([Char2Morse[ch] for ch in string.upper()])

if __name__ == '__main__':
    morse = '.'
    while morse:
        morse = input('Enter morse (...-), nothing to exit: ')
        result = morse2display(morse)
        print(result)
<commit_msg>Fixed error calculating stats for new char (0 samples)<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Small utility functions.
"""

import sys
import traceback
from random import betavariate
from random import choices
from math import floor

import logger

# various charsets
Alphabetics = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
Numbers = '0123456789'
Punctuation = """?,.!=/()'":;"""
AllUserChars = Alphabetics + Numbers + Punctuation

# all user chars in the 'Koch' order
Koch = """KMRSUAPTLOWI.NJE=F0Y,VG5/Q9ZH38B?427C1D6X():;!"'"""

# ensure Koch is same as AllUserChars
auc_sorted = ''.join(sorted(AllUserChars))
koch_sorted = ''.join(sorted(Koch))
if auc_sorted != koch_sorted:
    print('*'*80)
    print('* Error in utils.py, Koch and AllUserChars are different')
    print('*     len(Koch)=%d' % len(Koch))
    print('*     len(AllUserChars)=%d' % len(AllUserChars))
    print('*     auc_sorted= %s' % auc_sorted)
    print('*     koch_sorted=%s' % koch_sorted)
    print('*'*80)
    sys.exit(1)

# map chars to morse strings
Char2Morse = {
              '!': '-.-.--', '"': '.-..-.', '$': '...-..-', '&': '.-...',
              "'": '.----.', '(': '-.--.', ')': '-.--.-', ',': '--..--',
              '-': '-....-', '.': '.-.-.-', '/': '-..-.', ':': '---...',
              ';': '-.-.-.', '=': '-...-', '?': '..--..', '@': '.--.-.',
              '_': '..--.-', '+': '.-.-.',

              '0': '-----', '1': '.----', '2': '..---', '3': '...--',
              '4': '....-', '5': '.....', '6': '-....', '7': '--...',
              '8': '---..', '9': '----.',

              'A': '.-', 'B': '-...', 'C': '-.-.', 'D': '-..',
              'E': '.', 'F': '..-.', 'G': '--.', 'H': '....',
              'I': '..', 'J': '.---', 'K': '-.-', 'L': '.-..',
              'M': '--', 'N': '-.', 'O': '---', 'P': '.--.',
              'Q': '--.-', 'R': '.-.', 'S': '...', 'T': '-',
              'U': '..-', 'V': '...-', 'W': '.--', 'X': '-..-',
              'Y': '-.--', 'Z': '--..',

              ' ': ' '
             }

# invert above dict to create map of morse strings to chars
Morse2Char = {v:k for (k, v) in Char2Morse.items()}

# stylesheet code for PyQt5
#StyleCSS = ""
StyleCSS = """
/*css stylesheet file that contains all the style information*/

table, th, td {
    border: 1px solid black;
    border-collapse: collapse;
}

QGroupBox {
    border: 1px solid black;
    border-radius: 3px;
}

QGroupBox:title{
    subcontrol-origin: margin;
    subcontrol-position: top left;
    padding-left: 0px;
    padding-right: 5px;
    padding-top: -7px;
    border-top: none;
}
"""

log = logger.Log('debug.log', logger.Log.CRITICAL)

def str_trace(msg=None):
    """Get a traceback string.

    This is useful if we need at any point in code to find out how
    we got to that point.
    """

    result = []

    if msg:
        result.append(msg+'\n')

    result.extend(traceback.format_stack())

    return ''.join(result)

def log_trace(msg=None):
    """Log a traceback string."""

    log.debug(str_trace(msg))

def morse2display(morse):
    """Convert a string for a morse character to 'display' morse.

    We use this to display expected/received characters in the
    display as using ./- is not nice.  For example, '.-.' is not
    as nice as '• ━ •'
    """

    # unicode chars
    DOT = '•'    #2022
    DASH = '━'   #2501
    SIX_PER_EM_SPACE = ' '   #2005

    result = []
    for dotdash in morse:
        if dotdash == '.':
            result.append(DOT)
        elif dotdash in '-_':
            result.append(DASH)
        elif dotdash == ' ':
            result.append(' ')
        else:
            raise Exception("Unrecognized sign in '%s': %s" % (morse, dotdash))

    return SIX_PER_EM_SPACE.join(result)

def get_random_char(charset, stats):
    """Get a random char from the charset sequence.

    charset  a string of characters we are testing
    stats    a dictionary of stats: {'A': [T, F, T, ...], ...}

    Choose a character biased more towards the characters most in error.
    We do this by sorting the charset by error rate, then choosing from the
    front of the sorted list.

    A low number of samples biases character(s) to be more frequent.
    """

    # figure out the error rate of chars in the charset
    test_stats = {}
    for ch in charset:
        test_stats[ch] = stats[ch]
    ordered_charset = stats2errorrate(test_stats)

    bias = 0.5
    charset_len = len(charset)
    weights = [x+bias for x in range(charset_len)]
    sum_weights = sum(weights)
    normal_weights = [x/sum_weights for x in weights]

    return choices(ordered_charset, weights=normal_weights)[0]

def old_get_random_char(charset, stats):
    """Get a random char from the charset sequence.

    charset  a string of characters we are testing
    stats    a dictionary of stats: {'A': [T, F, T, ...], ...}

    Choose a character biased more towards the characters most in error.
    We do this by sorting the charset by error rate, then choosing from the
    front of the sorted list.
    """

    # figure out the error rate of chars in the charset
    test_stats = {}
    for ch in charset:
        test_stats[ch] = stats[ch]
    weighted_charset = stats2errorrate(test_stats)

    # choose randomly, but more likely the erroring char(s)
    beta = betavariate(1, 3)
    beta_len = beta * len(charset)
    rand_sample = floor(beta_len)
    result = weighted_charset[rand_sample]

    return result

def stats2errorrate(stats):
    """Convert stats data into an errorrate list.

    stats      dictionary holding statistics data

    The 'stats' data has the form {'A':[T,F,T], ...} where
    the list contains the last N results (True or False).

    The result is a list of characters sorted so the character with
    the highest error rate is forst, etc.
    """

    # working list of tuples (rate, char)
    temp = []

    # get a list of (rate, char)
    for (char, result_list) in stats.items():
        sample_size = len(result_list)
        try:
            rate = result_list.count(True) / sample_size
        except ZeroDivisionError:
            # sample_size is zero, allocate low proficiency
            rate = 0.01
        if len(result_list) < 50:
            # if few results yet, test mostest, probably just added
            rate *= len(result_list)/50

        temp.append((rate, char))

    # sort by error rate, drop the rates
    temp2 = sorted(temp, key=lambda t: t[0], reverse=True)
    result = [ch for (_, ch) in temp2]

    return result

def char2morse(char):
    """Convert a character into a morse string."""

    return morse2display(Char2Morse[char.upper()])

def str2morse(string):
    """Convert a string into a morse string.

    Spaces are inserted between each character in morse.
    """

    return ' '.join([Char2Morse[ch] for ch in string.upper()])

if __name__ == '__main__':
    morse = '.'
    while morse:
        morse = input('Enter morse (...-), nothing to exit: ')
        result = morse2display(morse)
        print(result)
<|endoftext|>


<commit_before>import sublime
import sys

def plugin_loaded():
    rsettings = sublime.load_settings('R-Box.sublime-settings')
    ssettings = sublime.load_settings('SendText+.sublime-settings')
    psettings = sublime.load_settings('Preferences.sublime-settings')

    if not rsettings.get("show_sendtext_installer_message", True):
        return

    if "SendText+" in psettings.get("ignored_packages", []):
        return

    if "SendText+" in sys.modules or "SendTextPlus" in sys.modules:
        return

    if "package_control" not in sys.modules:
        return

    ok = sublime.ok_cancel_dialog(
        "R-Box: \"SendText+\" is missing, "
        "sending code feature requires \"SendText+\". Install it now?")
    if ok:
        prog = rsettings.get("prog")
        if prog:
            ssettings.set("prog", prog)
            sublime.save_settings('SendText+.sublime-settings')
        sublime.active_window().run_command(
            "advanced_install_package", {"packages": "SendText+"})
<commit_msg>use sendtextplus<commit_after>import sublime
import sys

def plugin_loaded():
    rsettings = sublime.load_settings('R-Box.sublime-settings')
    ssettings = sublime.load_settings('SendText+.sublime-settings')
    psettings = sublime.load_settings('Preferences.sublime-settings')

    if not rsettings.get("show_sendtext_installer_message", True):
        return

    if "SendText+" in psettings.get("ignored_packages", []) or \
            "SendTextPlus" in psettings.get("ignored_packages", []):
        return

    if "SendText+" in sys.modules or "SendTextPlus" in sys.modules:
        return

    if "package_control" not in sys.modules:
        return

    ok = sublime.ok_cancel_dialog(
        "R-Box: \"SendTextPlus\" is missing, "
        "sending code feature requires \"SendTextPlus\". Install it now?")
    if ok:
        prog = rsettings.get("prog")
        if prog:
            ssettings.set("prog", prog)
            sublime.save_settings('SendText+.sublime-settings')
        sublime.active_window().run_command(
            "advanced_install_package", {"packages": "SendTextPlus"})
<|endoftext|>