<commit_before>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#
import copy
import os
from pprint import pprint

import ray
import ray.resource_spec
from ray import ray_constants
from ray.tune import Trainable, TuneError, tune
from ray.tune.ray_trial_executor import RESOURCE_REFRESH_PERIOD, RayTrialExecutor
from ray.tune.resources import Resources

from nupic.research.frameworks.pytorch.imagenet import ImagenetExperiment

os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"

def _get_resources_per_node():
    """
    Maps node id to available resources on that node
    :return: dict with available :class:`Resources` for each node
    """
    def _is_node_key(k):
        return k.startswith(ray.resource_spec.NODE_ID_PREFIX)

    # Only consider active/alive nodes
    nodes = filter(lambda node: node["Alive"], ray.nodes())
    resources = map(lambda node: node["Resources"], nodes)

    # Group resources by node
    resources_by_node = {}
    for item in resources:
        node_id = next(filter(_is_node_key, item.keys()))

        item = item.copy()
        num_cpus = item.pop("CPU", 0)
        num_gpus = item.pop("GPU", 0)
        memory = ray_constants.from_memory_units(item.pop("memory", 0))
        object_store_memory = ray_constants.from_memory_units(
            item.pop("object_store_memory", 0))
        custom_resources = item

        resources_by_node[node_id] = Resources(
            int(num_cpus),
            int(num_gpus),
            memory=int(memory),
            object_store_memory=int(object_store_memory),
            custom_resources=custom_resources)

    return resources_by_node

class AffinityExecutor(RayTrialExecutor):
    """
    Ray Trial Executor used to set node affinity to node trials.
    This trial executor will update the trial configuration with one extra
    parameter (__ray_node_affinity__) that can be used to launch extra workers
    on the same node.
    """

    def __init__(self, queue_trials=False, reuse_actors=False, ray_auto_init=False,
                 refresh_period=RESOURCE_REFRESH_PERIOD):
        self._resources_by_node = {}
        super().__init__(queue_trials, reuse_actors, ray_auto_init, refresh_period)

    def _commit_resources(self, resources):
        resources = copy.deepcopy(resources)
        # Make sure "custom_resources" keys match "extra_custom_resources" keys
        # because `RayTrialExecutor._commit_resources` will only commit
        # resources from "extra_custom_resources" if the same resource key
        # is present in the "custom_resources" dict
        for k in resources.extra_custom_resources:
            if k not in resources.custom_resources:
                resources.custom_resources[k] = 0.0

        super()._commit_resources(resources)

    def _update_avail_resources(self, num_retries=5):
        super()._update_avail_resources(num_retries)
        self._resources_by_node = _get_resources_per_node()

    def start_trial(self, trial, checkpoint=None):
        # Reserve node before starting trial
        resources = trial.resources

        # Check for required GPUs first
        required = resources.gpu_total()
        if required > 0:
            resource_attr = "gpu"
        else:
            # No GPU, just use CPU
            resource_attr = "cpu"
            required = resources.cpu_total()

        # Compute nodes required to fulfill trial resources request
        custom_resources = {}
        for node_id, node_resource in self._resources_by_node.items():
            # Compute resource remaining on each node
            node_capacity = node_resource.get_res_total(node_id)
            committed_capacity = self._committed_resources.get_res_total(node_id)
            remaining_capacity = node_capacity - committed_capacity
            node_procs = getattr(node_resource, resource_attr, 0)
            available = node_procs * remaining_capacity

            if available == 0:
                continue

            if required <= available:
                custom_resources[node_id] = required / node_procs
                required = 0
                break
            else:
                custom_resources[node_id] = remaining_capacity
                required -= available

        if required > 0:
            raise TuneError(f"Unable to start trial {trial.trial_id}. "
                            f"Not enough nodes")

        # Update trial node affinity configuration and
        # extra_custom_resources requirement
        trial.config.update(__ray_node_affinity__=custom_resources)
        trial.resources.extra_custom_resources.update(custom_resources)
        super().start_trial(trial, checkpoint)

    def has_resources(self, resources):
        if not super().has_resources(resources):
            return False

        # Check for required GPUs first
        required = resources.gpu_total()
        if required > 0:
            resource_attr = "gpu"
        else:
            # No GPU, just use CPU
            resource_attr = "cpu"
            required = resources.cpu_total()

        # Compute nodes required to fulfill trial resources request
        for node_id, node_resource in self._resources_by_node.items():
            node_capacity = node_resource.get_res_total(node_id)
            committed_capacity = self._committed_resources.get_res_total(node_id)
            remaining_capacity = node_capacity - committed_capacity
            node_procs = getattr(node_resource, resource_attr, 0)
            available = node_procs * remaining_capacity
            required -= available
            if required <= 0:
                return True

        return False

class ImagenetTrainable(Trainable):
    """
    Trainable class used to train resnet50 on Imagenet dataset using ray
    """

    @classmethod
    def default_resource_request(cls, config):
        """
        Configure the cluster resources used by this experiment
        """
        num_gpus = config.get("num_gpus", 0)
        num_cpus = config.get("num_cpus", 1)

        if num_gpus > 0:
            # Assign extra CPUs for dataloaders
            workers = config.get("workers", 0)
            num_cpus = workers * num_gpus

        resource = Resources(cpu=0, gpu=0, extra_cpu=num_cpus, extra_gpu=num_gpus)
        return resource

    def _setup(self, config):
        num_gpus = config.get("num_gpus", 0)
        num_cpus = config.get("num_cpus", 1)

        # Determine the number of distributed processes based on the number
        # GPUs and CPUs
        if num_gpus > 0:
            resource_attr = "gpu"
            world_size = num_gpus
            # Assign one GPU per remote process
            num_gpus = 1
            # Assign extra CPUs for dataloaders
            num_cpus = config.get("workers", 0)
        else:
            resource_attr = "cpu"
            world_size = num_cpus
            # Assign one CPU per remote process
            num_cpus = 1

        # Check for node affinity
        resources_by_node = _get_resources_per_node()
        ray_node_affinity = config.get("__ray_node_affinity__", {})
        node_resource = 0.0
        committed_resource = 0.0
        resources = {}

        # Create one ray remote process for each experiment in the process group
        self.procs = []
        for _ in range(world_size):
            if ray_node_affinity and node_resource <= 0:
                node_id, node_resource = ray_node_affinity.popitem()
                proc_per_node = getattr(resources_by_node[node_id], resource_attr, 0)
                committed_resource = 1.0 / proc_per_node
                resources = {node_id: committed_resource}

            experiment = ray.remote(num_cpus=num_cpus, num_gpus=num_gpus,
                                    resources=resources)(ImagenetExperiment)
            self.procs.append(experiment.remote())
            node_resource -= committed_resource

        # Use first process as head of the group
        ip = ray.get(self.procs[0].get_node_ip.remote())
        port = config.get("dist_port", 54321)
        dist_url = "tcp://{}:{}".format(ip, port)

        # Configure each process in the group
        status = []
        for i, w in enumerate(self.procs):
            worker_config = copy.deepcopy(config)
            worker_config["distributed"] = True
            worker_config["dist_url"] = dist_url
            worker_config["world_size"] = world_size
            worker_config["rank"] = i
            status.append(w.setup_experiment.remote(worker_config))

        # Wait for remote functions to complete
        ray.get(status)

    def _train(self):
        status = []
        for w in self.procs:
            status.append(w.run_epoch.remote(self.iteration))

        # Wait for remote functions to complete
        results = ray.get(status)

        # Return the results from the first remote function
        return copy.deepcopy(results[0])

    def _save(self, _):
        # All models are synchronized. Just save the state of first model
        return ray.get(self.procs[0].get_state.remote())

    def _restore(self, state):
        # Restore the state to every process
        state_id = ray.put(state)
        ray.get([w.set_state.remote(state_id) for w in self.procs])

    def _stop(self):
        for w in self.procs:
            w.stop_experiment.remote()
            w.__ray_terminate__.remote()

def run(config):
    # Connect to ray
    address = os.environ.get("REDIS_ADDRESS", config.get("redis_address"))
    ray.init(address=address)

    # Build kwargs for `tune.run` function using merged config and command line dict
    kwargs_names = tune.run.__code__.co_varnames[:tune.run.__code__.co_argcount]
    kwargs = dict(zip(kwargs_names, [ImagenetTrainable, *tune.run.__defaults__]))

    # Update`tune.run` kwargs with config
    kwargs.update(config)
    kwargs["config"] = config

    # Update tune stop criteria with config epochs
    stop = kwargs.get("stop", {}) or dict()
    epochs = config.get("epochs", 1)
    stop.update(training_iteration=epochs)
    kwargs["stop"] = stop

    # Make sure to only select`tune.run` function arguments
    kwargs = dict(filter(lambda x: x[0] in kwargs_names, kwargs.items()))

    # Queue trials until the cluster scales up
    kwargs.update(queue_trials=True)

    # Group trial into nodes as much as possible
    kwargs.update(trial_executor=AffinityExecutor(
        queue_trials=kwargs.get("queue_trials", True),
        reuse_actors=kwargs.get("reuse_actors", False),
        ray_auto_init=kwargs.get("ray_auto_init", True)
    ))

    pprint(kwargs)
    tune.run(**kwargs)
    ray.shutdown()
<commit_msg>RES-1267: Fix ray autoscaler with node affinity<commit_after>#  Numenta Platform for Intelligent Computing (NuPIC)
#  Copyright (C) 2019, Numenta, Inc.  Unless you have an agreement
#  with Numenta, Inc., for a separate license for this software code, the
#  following terms and conditions apply:
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero Public License version 3 as
#  published by the Free Software Foundation.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#  See the GNU Affero Public License for more details.
#
#  You should have received a copy of the GNU Affero Public License
#  along with this program.  If not, see http://www.gnu.org/licenses.
#
#  http://numenta.org/licenses/
#
import copy
import logging
import os
import traceback
from pprint import pprint

import ray
import ray.resource_spec
from ray import ray_constants
from ray.exceptions import RayTimeoutError
from ray.tune import Trainable, tune
from ray.tune.ray_trial_executor import RESOURCE_REFRESH_PERIOD, RayTrialExecutor
from ray.tune.resources import Resources

from nupic.research.frameworks.pytorch.imagenet import ImagenetExperiment

os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE"
logger = logging.getLogger(__name__)
TRIAL_START_ATTEMPTS = 3
NODE_CREATION_TIMEOUT = 10 * 60

def _get_resources_per_node():
    """
    Maps node id to available resources on that node
    :return: dict with available :class:`Resources` for each node
    """
    def _is_node_key(k):
        return k.startswith(ray.resource_spec.NODE_ID_PREFIX)

    # Only consider active/alive nodes
    nodes = filter(lambda node: node["Alive"], ray.nodes())
    resources = map(lambda node: node["Resources"], nodes)

    # Group resources by node
    resources_by_node = {}
    for item in resources:
        node_id = next(filter(_is_node_key, item.keys()))

        item = item.copy()
        num_cpus = item.pop("CPU", 0)
        num_gpus = item.pop("GPU", 0)
        memory = ray_constants.from_memory_units(item.pop("memory", 0))
        object_store_memory = ray_constants.from_memory_units(
            item.pop("object_store_memory", 0))
        custom_resources = item

        resources_by_node[node_id] = Resources(
            int(num_cpus),
            int(num_gpus),
            memory=int(memory),
            object_store_memory=int(object_store_memory),
            custom_resources=custom_resources)

    return resources_by_node

class AffinityExecutor(RayTrialExecutor):
    """
    Ray Trial Executor used to set node affinity to node trials.
    This trial executor will update the trial configuration with one extra
    parameter (__ray_node_affinity__) that can be used to launch extra workers
    on the same node.
    """

    def __init__(self, queue_trials=False, reuse_actors=False, ray_auto_init=False,
                 refresh_period=RESOURCE_REFRESH_PERIOD):
        self._resources_by_node = {}
        super().__init__(queue_trials, reuse_actors, ray_auto_init, refresh_period)

    def _commit_resources(self, resources):
        resources = copy.deepcopy(resources)
        # Make sure "custom_resources" keys match "extra_custom_resources" keys
        # because `RayTrialExecutor._commit_resources` will only commit
        # resources from "extra_custom_resources" if the same resource key
        # is present in the "custom_resources" dict
        for k in resources.extra_custom_resources:
            if k not in resources.custom_resources:
                resources.custom_resources[k] = 0.0

        super()._commit_resources(resources)

    def _update_avail_resources(self, num_retries=5):
        super()._update_avail_resources(num_retries)
        self._resources_by_node = _get_resources_per_node()

    def _get_node_resources(self, resources):
        self._update_avail_resources()

        # Check for required GPUs first
        required = resources.gpu_total()
        if required > 0:
            resource_attr = "gpu"
        else:
            # No GPU, just use CPU
            resource_attr = "cpu"
            required = resources.cpu_total()

        # Compute nodes required to fulfill trial resources request
        custom_resources = {}
        for node_id, node_resource in self._resources_by_node.items():
            # Compute resource remaining on each node
            node_capacity = node_resource.get_res_total(node_id)
            committed_capacity = self._committed_resources.get_res_total(node_id)
            remaining_capacity = node_capacity - committed_capacity
            node_procs = getattr(node_resource, resource_attr, 0)
            available = node_procs * remaining_capacity

            if available == 0:
                continue

            if required <= available:
                custom_resources[node_id] = required / node_procs
                required = 0
                break
            else:
                custom_resources[node_id] = remaining_capacity
                required -= available

        if required > 0:
            # Not enough nodes
            return None

        return custom_resources

    def start_trial(self, trial, checkpoint=None):
        # Reserve node before starting trial
        resources = trial.resources

        # Use no-op remote with same resource requirements as trial to trick
        # ray into scale up the cluster before we allocate the node resources
        # for the trial
        wait_for_resources = ray.remote(
            num_cpus=resources.cpu_total(),
            num_gpus=resources.gpu_total())(lambda: None)

        error_msg = None
        for _ in range(TRIAL_START_ATTEMPTS):
            try:
                # Wait for ray to scale up the cluster
                wait_status = wait_for_resources.remote()
                ray.get(wait_status, NODE_CREATION_TIMEOUT)

                custom_resources = self._get_node_resources(resources)
                if custom_resources is not None:
                    break
                logger.warning("Trial %s: Not enough nodes, retrying...", trial)
            except RayTimeoutError:
                logger.warning("Trial %s: Timed out, retrying...", trial)
            except Exception as ex:
                logger.exception("Trial %s: Error starting trial, aborting!", trial)
                error_msg = traceback.format_exception(ex)
                break
        else:
            logger.exception(
                "Trial %s: Aborting trial after %s start "
                "attempts!", trial, TRIAL_START_ATTEMPTS)
            self._stop_trial(trial, error=True, error_msg=error_msg)
            return

        # Update trial node affinity configuration and
        # extra_custom_resources requirement
        trial.config.update(__ray_node_affinity__=custom_resources)
        trial.resources.extra_custom_resources.update(custom_resources)
        super().start_trial(trial, checkpoint)

    def has_resources(self, resources):
        if not super().has_resources(resources):
            return False
        elif self._trial_queued:
            # Allowing trial to start even though the cluster does not have
            # enough free resources
            return True

        node_resources = self._get_node_resources(resources)
        if node_resources is not None:
            return True

        if self._queue_trials:
            self._trial_queued = True
            logger.warning(
                "Allowing trial to start even though the "
                "cluster does not have enough free resources. Trial actors "
                "may appear to hang until enough resources are added to the "
                "cluster (e.g., via autoscaling). You can disable this "
                "behavior by specifying `queue_trials=False` in "
                "ray.tune.run().")
            return True
        return False

class ImagenetTrainable(Trainable):
    """
    Trainable class used to train resnet50 on Imagenet dataset using ray
    """

    @classmethod
    def default_resource_request(cls, config):
        """
        Configure the cluster resources used by this experiment
        """
        num_gpus = config.get("num_gpus", 0)
        num_cpus = config.get("num_cpus", 1)

        if num_gpus > 0:
            # Assign extra CPUs for dataloaders
            workers = config.get("workers", 0)
            num_cpus = workers * num_gpus

        resource = Resources(cpu=0, gpu=0, extra_cpu=num_cpus, extra_gpu=num_gpus)
        return resource

    def _setup(self, config):
        num_gpus = config.get("num_gpus", 0)
        num_cpus = config.get("num_cpus", 1)

        # Determine the number of distributed processes based on the number
        # GPUs and CPUs
        if num_gpus > 0:
            resource_attr = "gpu"
            world_size = num_gpus
            # Assign one GPU per remote process
            num_gpus = 1
            # Assign extra CPUs for dataloaders
            num_cpus = config.get("workers", 0)
        else:
            resource_attr = "cpu"
            world_size = num_cpus
            # Assign one CPU per remote process
            num_cpus = 1

        # Check for node affinity
        resources_by_node = _get_resources_per_node()
        ray_node_affinity = config.get("__ray_node_affinity__", {})
        node_resource = 0.0
        committed_resource = 0.0
        resources = {}

        # Create one ray remote process for each experiment in the process group
        self.procs = []
        for _ in range(world_size):
            if ray_node_affinity and node_resource <= 0:
                node_id, node_resource = ray_node_affinity.popitem()
                proc_per_node = getattr(resources_by_node[node_id], resource_attr, 0)
                committed_resource = 1.0 / proc_per_node
                resources = {node_id: committed_resource}

            experiment = ray.remote(num_cpus=num_cpus, num_gpus=num_gpus,
                                    resources=resources)(ImagenetExperiment)
            self.procs.append(experiment.remote())
            node_resource -= committed_resource

        # Use first process as head of the group
        ip = ray.get(self.procs[0].get_node_ip.remote())
        port = config.get("dist_port", 54321)
        dist_url = "tcp://{}:{}".format(ip, port)

        # Configure each process in the group
        status = []
        for i, w in enumerate(self.procs):
            worker_config = copy.deepcopy(config)
            worker_config["distributed"] = True
            worker_config["dist_url"] = dist_url
            worker_config["world_size"] = world_size
            worker_config["rank"] = i
            status.append(w.setup_experiment.remote(worker_config))

        # Wait for remote functions to complete
        ray.get(status)

    def _train(self):
        status = []
        for w in self.procs:
            status.append(w.run_epoch.remote(self.iteration))

        # Wait for remote functions to complete
        results = ray.get(status)

        # Return the results from the first remote function
        return copy.deepcopy(results[0])

    def _save(self, _):
        # All models are synchronized. Just save the state of first model
        return ray.get(self.procs[0].get_state.remote())

    def _restore(self, state):
        # Restore the state to every process
        state_id = ray.put(state)
        ray.get([w.set_state.remote(state_id) for w in self.procs])

    def _stop(self):
        for w in self.procs:
            w.stop_experiment.remote()
            w.__ray_terminate__.remote()

def run(config):
    # Connect to ray
    address = os.environ.get("REDIS_ADDRESS", config.get("redis_address"))
    ray.init(address=address)

    # Build kwargs for `tune.run` function using merged config and command line dict
    kwargs_names = tune.run.__code__.co_varnames[:tune.run.__code__.co_argcount]
    kwargs = dict(zip(kwargs_names, [ImagenetTrainable, *tune.run.__defaults__]))

    # Update`tune.run` kwargs with config
    kwargs.update(config)
    kwargs["config"] = config

    # Update tune stop criteria with config epochs
    stop = kwargs.get("stop", {}) or dict()
    epochs = config.get("epochs", 1)
    stop.update(training_iteration=epochs)
    kwargs["stop"] = stop

    # Make sure to only select`tune.run` function arguments
    kwargs = dict(filter(lambda x: x[0] in kwargs_names, kwargs.items()))

    # Queue trials until the cluster scales up
    kwargs.update(queue_trials=True)

    # Group trial into nodes as much as possible
    kwargs.update(trial_executor=AffinityExecutor(
        queue_trials=kwargs.get("queue_trials", True),
        reuse_actors=kwargs.get("reuse_actors", False),
        ray_auto_init=kwargs.get("ray_auto_init", True)
    ))

    pprint(kwargs)
    tune.run(**kwargs)
    ray.shutdown()
<|endoftext|>


<commit_before># Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from platformio.commands.device import DeviceMonitorFilter

class SendOnEnter(DeviceMonitorFilter):
    NAME = "send_on_enter"

    def __init__(self, *args, **kwargs):
        super(SendOnEnter, self).__init__(*args, **kwargs)
        self._buffer = ""

    def tx(self, text):
        self._buffer += text
        if "\r\n" in self._buffer:
            text = self._buffer
            self._buffer = ""
            return text
        return ""
<commit_msg>Do not send CR+NL for "send_on_enter" device monitor filter<commit_after># Copyright (c) 2014-present PlatformIO <contact@platformio.org>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from platformio.commands.device import DeviceMonitorFilter

class SendOnEnter(DeviceMonitorFilter):
    NAME = "send_on_enter"

    def __init__(self, *args, **kwargs):
        super(SendOnEnter, self).__init__(*args, **kwargs)
        self._buffer = ""

    def tx(self, text):
        self._buffer += text
        if self._buffer.endswith("\r\n"):
            text = self._buffer[:-2]
            self._buffer = ""
            return text
        return ""
<|endoftext|>


<commit_before>from bencode import bencode, bdecode
from hashlib import md5, sha1
import requests
import random
import socket

import utils
from nest import Nest
from peer import Peer

nest = None

CLIENT_NAME = 'wasptorret'
VERSION_NUM = '0001'
CLIENT_ID = 'WS'

def main():
    parse_torrent()

def parse_torrent():
    """DESC: Asks user for location of .torrent file.
    Parses and saves metadata results"""

    t_content = ''
    meta_dict = {}
    global nest
    # torrent_file = input("Torrent File Location: ")
    torrent_file = '../test/tst.torrent'
    print(torrent_file)
    if nest is None:
        # TODO: check if nest saved to disk else create a new nest
        nest = Nest()

    try:
        # Attempt to open the torrent file and bdecode the metadata
        with open(torrent_file, 'rb') as content_file:
            t_content = content_file.read()
            meta_dict = bdecode(t_content)
    except IOError:
        print('ERROR: Could not open file: ' + torrent_file)
    except:
        print ('ERROR: An unknown error occurred in opening or decoding file.')
    else:
        nest.hatch(meta_dict)

class Wasp(object):
    """DESC: A wasp is a single instance of a torrent.
    Includes all metadata and functionality in seeding/leeching"""

    def __init__(self, meta_data):
        # TODO: Handle non-existant vars appropriately
        self.announce = meta_data[b'announce']
        self.piece_length = meta_data[b'info'][b'piece length']
        self.pieces = meta_data[b'info'][b'pieces']
        self.info_hash = sha1(bencode(meta_data[b'info'])).digest()
        self.name = meta_data[b'info'][b'name']
        self.peer_id = ''
        # TODO: Only works on single file torrents
        file = meta_data[b'info'][b'files'][0]
        self.length = file[b'length']
        self.peer_dict = {}

    def generate_peer_id(self):
        """DESC: Generate unique wasp peer id.
        Used as standard in bittorrent protocol."""

        wasp_random = str(random.randint(100000000000, 999999999999))
        return '-' + CLIENT_ID + VERSION_NUM + '-' + wasp_random

    def generate_handshake(self):
        """DESC: Generates and returns initial bit torrent handshake"""

        protocol_id = "BitTorrent protocol"
        id_length = len(protocol_id)
        if len(self.peer_id) == 0:
            self.generate_peer_id()

        # Initial handshake from docs
        return ''.join([chr(id_length), protocol_id,
                        chr(0) * 8, self.info_hash,
                        self.peer_id])

    def send_handshake(self, host, port):
        handshake = self.generate_handshake()
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((host, port))
        sock.send(handshake)

        data = sock.recv(len(handshake))
        try:
            data = sock.recv(68)  # Peer response handshake
            if data:
                print(data)
                # import pdb; pdb.set_trace()
                peer_id = md5(host + str(port)).hexdigest()
                temp_peer = Peer(sock, self, peer_id)
                self.peer_dict[peer_id] = temp_peer

                # self.initpeer(sock)
        except:
            print("ERROR: Did not recieve proper return handshake from peer")

    def apple(self):
        # I'm leaving this forever
        print("I AM CLASSY APPLES!")

    def get_peers(self):
        # TODO: For multi files, compute total length
        # from the individual file lengths
        length = self.length
        # Peer id consists of a client code(WS) with a version num and a
        # random client identifier 12 characters long hence the randomint range
        peer_id = self.generate_peer_id()
        self.peer_id = peer_id
        payload = {'info_hash': self.info_hash, 'peer_id': peer_id,
                   'left': length}
        req = requests.get(self.announce, params=payload)
        peers = utils.parse_peerlist(req)
        # TODO: send peers off to be handled
        # TODO: only doing handshake on first peer
        for peer in peers:
            if peer[1] < 1024:
                # Basic step to check if valid ip. If not skip. This should fix
                # the fact that you are included in the ip list
                # which is incorrect
                continue
            self.send_handshake(peer[0], peer[1])

if __name__ == "__main__":
    main()
<commit_msg>basics of wasp reformatted to fit python 3.5<commit_after>from bencode import bencode, bdecode
from hashlib import md5, sha1
import requests
import random
import socket

import utils
from nest import Nest
from peer import Peer

nest = None

CLIENT_NAME = 'wasptorret'
VERSION_NUM = '0001'
CLIENT_ID = 'WS'

def main():
    parse_torrent()

def parse_torrent():
    """DESC: Asks user for location of .torrent file.
    Parses and saves metadata results"""

    t_content = ''
    meta_dict = {}
    global nest
    # torrent_file = input("Torrent File Location: ")
    torrent_file = '../test/tst.torrent'
    print(torrent_file)
    if nest is None:
        # TODO: check if nest saved to disk else create a new nest
        nest = Nest()

    try:
        # Attempt to open the torrent file and bdecode the metadata
        with open(torrent_file, 'rb') as content_file:
            t_content = content_file.read()
            meta_dict = bdecode(t_content)
    except IOError:
        print('ERROR: Could not open file: ' + torrent_file)
    except:
        print ('ERROR: An unknown error occurred in opening or decoding file.')
    else:
        nest.hatch(meta_dict)

class Wasp(object):
    """DESC: A wasp is a single instance of a torrent.
    Includes all metadata and functionality in seeding/leeching"""

    def __init__(self, meta_data):
        # TODO: Handle non-existant vars appropriately
        self.announce = meta_data[b'announce']
        self.piece_length = meta_data[b'info'][b'piece length']
        self.pieces = meta_data[b'info'][b'pieces']
        self.info_hash = sha1(bencode(meta_data[b'info'])).digest()
        self.name = meta_data[b'info'][b'name']
        self.peer_id = ''
        # TODO: Only works on single file torrents
        file = meta_data[b'info'][b'files'][0]
        self.length = file[b'length']
        self.peer_dict = {}

    def generate_peer_id(self):
        """DESC: Generate unique wasp peer id.
        Used as standard in bittorrent protocol."""

        wasp_random = str(random.randint(100000000000, 999999999999))
        return bytes('-' + CLIENT_ID + VERSION_NUM + '-' + wasp_random, encoding='UTF-8')

    def generate_handshake(self):
        """DESC: Generates and returns initial bit torrent handshake"""

        protocol_id = b"BitTorrent protocol"
        id_length = len(protocol_id)
        if len(self.peer_id) == 0:
            self.generate_peer_id()

        # Initial handshake from docs
        return b''.join([(chr(id_length)).encode(), protocol_id,
                        (chr(0) * 8).encode(), self.info_hash,
                        self.peer_id])

    def send_handshake(self, host, port):
        handshake = self.generate_handshake()
        sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        sock.connect((host, port))
        sock.send(handshake)

        data = sock.recv(len(handshake))
        try:
            data = sock.recv(68)  # Peer response handshake
            if data:
                print(data)
                # import pdb; pdb.set_trace()
                # import pdb
                # pdb.set_trace()
                peer_id = md5(host.encode() +
                              str(port).encode()).hexdigest()
                temp_peer = Peer(sock, self, peer_id)
                self.peer_dict[peer_id] = temp_peer

                import pdb
                pdb.set_trace()
                # self.initpeer(sock)
        except:
            print("ERROR: Did not recieve proper return handshake from peer")

    def apple(self):
        # I'm leaving this forever
        print("I AM CLASSY APPLES!")

    def get_peers(self):
        # TODO: For multi files, compute total length
        # from the individual file lengths
        length = self.length
        # Peer id consists of a client code(WS) with a version num and a
        # random client identifier 12 characters long hence the randomint range
        peer_id = self.generate_peer_id()
        self.peer_id = peer_id
        payload = {'info_hash': self.info_hash, 'peer_id': peer_id,
                   'left': length}
        req = requests.get(self.announce, params=payload)
        peers = utils.parse_peerlist(req)
        # TODO: send peers off to be handled
        # TODO: only doing handshake on first peer
        for peer in peers:
            if peer[1] < 1024:
                # Basic step to check if valid ip. If not skip. This should fix
                # the fact that you are included in the ip list
                # which is incorrect
                continue
            self.send_handshake(peer[0], peer[1])

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""originally this was parseNamesGBIF.py

"Use the GBIF name parser API (http://tools.gbif.org/nameparser/api.do) to
disect [sic] name strings into their components. Input should be a simple list of
name strings separated by newline-characters. The names can be read either
from textfile(s) or from <STDIN>.

Output will be written as JSON to <STDOUT> by default.

Usage: parseNamesGBIF.py filename1 [filename2 [...]]

from:

https://www.snip2code.com/Snippet/162694/Parse-taxon-names-with-Python--using-the"

However, I have hacked it substantially for this UCBG application, and it now takes three command line arguments,
caches the results, etc. etc.

... jblowe@berkeley.edu 4/6/2015

"""

import fileinput
import pickle
import requests
import sys
import json
import time
import os
# import csv
import codecs

# empty class for counts
class count:
    pass

count.input = 0
count.output = 0
count.newnames = 0
count.source = 0
count.datasource = 0

parts = {}

nameparts = ["authorsParsed",
             "authorship",
             "bracketAuthorship",
             "canonicalName",
             "canonicalNameComplete",
             "canonicalNameWithMarker",
             "genusOrAbove",
             "infraSpecificEpithet",
             "rankMarker",
             "scientificName",
             "specificEpithet",
             "type"]

# from http://stackoverflow.com/questions/1158076/implement-touch-using-python
def touch(fname, times=None):
    with open(fname, 'a'):
        os.utime(fname, times)

def main():
    if len(sys.argv) < 4:
        print 'usage: %s inputfileofnames.csv outputnameparts.csv picklefile column' % sys.argv[0]
        sys.exit(1)

    column = 0
    try:
        column = int(sys.argv[4])
    except:
        print "column is not an integer: %s " % column
        sys.exit(1)

    try:
        namepartsfile = sys.argv[2]
        # namepartsfh = csv.writer(open(namepartsfile, "wb"), delimiter='\t')
        namepartsfh = open(namepartsfile, "wb")
        #namepartsfh.write('\t'.join(nameparts) + '\n')
    except:
        print "could not open output file"
        sys.exit(1)

    try:
        picklefile = sys.argv[3]
        picklefh = open(picklefile, "rb")
    except:
        print "could not open pickle file, will try to create"
        pickle.dump({}, open(picklefile, "wb"))

    try:
        parsednames = pickle.load(picklefh)
        picklefh.close()
        print "%s names in datasource." % len(parsednames.keys())
    except:
        raise
        print "could not parse data in picklefile %s" % picklefile
        sys.exit(1)

    try:
        inputfile = codecs.open(sys.argv[1], "rb", "utf-8")
    except:
        raise
        print "could not open input file %s" % sys.argv[1]
        sys.exit(1)

    for line in inputfile:
        count.input += 1
        inputrow = line.rstrip('\n')
        cells = inputrow.split('\t')
        name = cells[column]
        if name in parsednames:
            count.source += 1
            name2use = parsednames[name]
            pass
        else:
            time.sleep(1)  # delays for 1 second
            response = requests.get('http://api.gbif.org/v1/parser/name', params={'name': name})
            response.encoding = 'utf-8'
            name2use = response.json()[0]
            count.newnames += 1
            parsednames[name] = name2use

        row = []

        for part in name2use.keys():
            parts[part] = parts.get(part, 0) + 1

        for part in nameparts:
            if part in name2use:
                try:
                    row.append(name2use[part].encode('utf-8'))
                except:
                    row.append(str(name2use[part]))
            else:
                row.append('')

        if count.input == 1:
            row = [h + '_s' for h in nameparts]
        cells = [x.encode('utf-8') for x in cells]
        cells = cells[:column] + row + cells[column:]
        namepartsfh.write('\t'.join(cells) + '\n')

    try:
        pickle.dump(parsednames, open(picklefile, "wb"))
        count.datasource = len(parsednames.keys())
    except:
        print "could not write names to picklefile %s" % picklefile
        sys.exit(1)

    print "%s names input." % count.input
    print "%s parsenames output." % count.output
    print "%s new names found." % count.newnames
    print "%s names now in datasource." % count.datasource

    print
    print 'name parts:'
    for p in parts.keys():
        print "%s: %s" % (p, parts[p])

if __name__ == '__main__':
    main()

<commit_msg>DJAN-30: add 'cv.' where needed to improve parsing<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""originally this was parseNamesGBIF.py

"Use the GBIF name parser API (http://tools.gbif.org/nameparser/api.do) to
disect [sic] name strings into their components. Input should be a simple list of
name strings separated by newline-characters. The names can be read either
from textfile(s) or from <STDIN>.

Output will be written as JSON to <STDOUT> by default.

Usage: parseNamesGBIF.py filename1 [filename2 [...]]

from:

https://www.snip2code.com/Snippet/162694/Parse-taxon-names-with-Python--using-the"

However, I have hacked it substantially for this UCBG application, and it now takes three command line arguments,
caches the results, etc. etc.

... jblowe@berkeley.edu 4/6/2015

"""

import fileinput
import pickle
import requests
import re
import sys
import json
import time
import os
# import csv
import codecs

# empty class for counts
class count:
    pass

count.input = 0
count.output = 0
count.newnames = 0
count.source = 0
count.datasource = 0

parts = {}

nameparts = ["authorsParsed",
             "authorship",
             "bracketAuthorship",
             "canonicalName",
             "canonicalNameComplete",
             "canonicalNameWithMarker",
             "genusOrAbove",
             "infraSpecificEpithet",
             "rankMarker",
             "scientificName",
             "specificEpithet",
             "type"]

# from http://stackoverflow.com/questions/1158076/implement-touch-using-python
def touch(fname, times=None):
    with open(fname, 'a'):
        os.utime(fname, times)

# look for cultivars, e.g. "Ceanothus 'Berkeley Skies'", make it Ceanothus cv. 'Berkeley Skies' for GBIF parsing"
cultivarpattern = re.compile("(.*)('.*')")

def check4cultivars(name):
    if not 'cv.' in name:
        name = cultivarpattern.sub(r'\1 cv. \2', name)
    return name

def main():
    if len(sys.argv) < 4:
        print 'usage: %s inputfileofnames.csv outputnameparts.csv picklefile column' % sys.argv[0]
        sys.exit(1)

    namecolumn = 0
    try:
        namecolumn = int(sys.argv[4])
    except:
        print "column is not an integer: %s " % sys.argv[4]
        sys.exit(1)

    try:
        namepartsfile = sys.argv[2]
        # namepartsfh = csv.writer(open(namepartsfile, "wb"), delimiter='\t')
        namepartsfh = open(namepartsfile, "wb")
        #namepartsfh.write('\t'.join(nameparts) + '\n')
    except:
        print "could not open output file"
        sys.exit(1)

    try:
        picklefile = sys.argv[3]
        picklefh = open(picklefile, "rb")
    except:
        print "could not open pickle file, will try to create"
        picklefh = open(picklefile, "wb")
        pickle.dump({}, picklefh)
        picklefh.close()
        picklefh = open(picklefile, "rb")

    try:
        parsednames = pickle.load(picklefh)
        picklefh.close()
        print "%s names in datasource." % len(parsednames.keys())
    except:
        raise
        print "could not parse data in picklefile %s" % picklefile
        sys.exit(1)

    try:
        inputfile = codecs.open(sys.argv[1], "rb", "utf-8")
    except:
        raise
        print "could not open input file %s" % sys.argv[1]
        sys.exit(1)

    for line in inputfile:
        count.input += 1
        inputrow = line.rstrip('\n')
        cells = inputrow.split('\t')
        name = cells[namecolumn]
        # handle cultivars without 'cv.'...
        name = check4cultivars(name)
        if name in parsednames:
            count.source += 1
            name2use = parsednames[name]
        else:
            time.sleep(1)  # delays for 1 second
            response = requests.get('http://api.gbif.org/v1/parser/name', params={'name': name})
            response.encoding = 'utf-8'
            name2use = response.json()[0]
            count.newnames += 1
            parsednames[name] = name2use

        row = []

        for part in name2use.keys():
            parts[part] = parts.get(part, 0) + 1

        for part in nameparts:
            if part in name2use:
                try:
                    row.append(name2use[part].encode('utf-8'))
                except:
                    row.append(str(name2use[part]))
            else:
                row.append('')

        if count.input == 1:
            row = [h + '_s' for h in nameparts]
        cells = [x.encode('utf-8') for x in cells]
        cells = cells[:namecolumn] + row + cells[namecolumn:]
        namepartsfh.write('\t'.join(cells) + '\n')

    try:
        pickle.dump(parsednames, open(picklefile, "wb"))
        count.datasource = len(parsednames.keys())
    except:
        print "could not write names to picklefile %s" % picklefile
        sys.exit(1)

    print "%s names input." % count.input
    print "%s parsenames output." % count.output
    print "%s new names found." % count.newnames
    print "%s names now in datasource." % count.datasource

    print
    print 'name parts:'
    for p in parts.keys():
        print "%s: %s" % (p, parts[p])

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>#!/usr/bin/python3

import ID3
import os
import sys

def read_tracklist():
    tracklist = []
    for line in sys.stdin:
        tracklist.append(line)
    return tracklist

tracklist = read_tracklist()
mp3_extension = ".mp3"

files_all = os.listdir('.')
files = []

for f in files_all:

    # Prune directories
    if not os.path.isfile(f):
        continue

    # Prune non-MP3 files
    filename, extension = os.path.splitext(f)
    if extension != mp3_extension:
        continue

    # Prune this file
    f_temp = os.path.abspath(f)
    if f_temp == os.path.abspath(__file__):
        continue

    files.append(f)

if len(files) != len(tracklist):
    raise RuntimeError(
        str(len(tracklist)) +
        " file names were given but " +
        str(len(files)) +
        " files were found.")
    sys.exit()

files.sort()

i = 0
for f in files:
    os.rename(f, tracklist[i] + mp3_extension)
    i += 1
<commit_msg>MP3: Move files/tracklist count check to function<commit_after>#!/usr/bin/python3

import ID3
import os
import sys

def read_tracklist():
    tracklist = []
    for line in sys.stdin:
        tracklist.append(line)
    return tracklist

def match_length(files, tracklist):
    if len(files) != len(tracklist):
        raise RuntimeError(
            str(len(tracklist)) +
            " file names were given but " +
            str(len(files)) +
            " files were found.")

tracklist = read_tracklist()
mp3_extension = ".mp3"

files_all = os.listdir('.')
files = []

for f in files_all:

    # Prune directories
    if not os.path.isfile(f):
        continue

    # Prune non-MP3 files
    filename, extension = os.path.splitext(f)
    if extension != mp3_extension:
        continue

    # Prune this file
    f_temp = os.path.abspath(f)
    if f_temp == os.path.abspath(__file__):
        continue

    files.append(f)

match_length(files, tracklist)

files.sort()

i = 0
for f in files:
    os.rename(f, tracklist[i] + mp3_extension)
    i += 1
<|endoftext|>


<commit_before>#TODO: improve this

class MemeAPIError(Exception):
    pass

class MemeAPIBadFormatResponseError(Exception):
    pass
<commit_msg>More descriptive exceptions.<commit_after>class BadStatusError(Exception):
    pass

class NotJSONError(Exception):
    pass

class LanguageCodeError(Exception):
    pass
<|endoftext|>


<commit_before>"""
Misc cog.
"""
import asyncio
from itertools import islice

import discord
import google
import time

from asyncio_extras import threadpool
from discord.ext import commands

from joku.bot import Context
from joku.cogs._common import Cog

class Misc(Cog):
    @commands.command()
    async def ping(self, ctx: Context):
        """
        Pong.
        """
        before = time.monotonic()
        msg = await ctx.send(":ping_pong: Ping!")  # type: discord.Message
        after = time.monotonic()
        await msg.edit(content=":ping_pong: Ping! | {}ms".format(round((after - before) * 1000, 2)))

    @commands.command()
    async def cleanup(self, ctx: Context):
        """
        Cleans up bot messages in the last 100 messages.
        """
        channel = ctx.channel  # type: discord.TextChannel
        if channel.permissions_for(ctx.guild.me).manage_messages:
            count = len(await channel.purge(limit=100, check=lambda m: m.author == ctx.guild.me, before=ctx.message))
        else:
            count = 0
            async for message in channel.history(limit=100, before=ctx.message):
                if message.author == ctx.guild.me:
                    await message.delete()
                    count += 1

        m = await ctx.send(":heavy_check_mark: Cleaned up `{}` messages.".format(count))
        await asyncio.sleep(5)
        await m.delete()

setup = Misc.setup
<commit_msg>Remove dead imports<commit_after>"""
Misc cog.
"""
import asyncio
import time

import discord
from asyncio_extras import threadpool
from discord.ext import commands

from joku.bot import Context
from joku.cogs._common import Cog

class Misc(Cog):
    @commands.command()
    async def ping(self, ctx: Context):
        """
        Pong.
        """
        before = time.monotonic()
        msg = await ctx.send(":ping_pong: Ping!")  # type: discord.Message
        after = time.monotonic()
        await msg.edit(content=":ping_pong: Ping! | {}ms".format(round((after - before) * 1000, 2)))

    @commands.command()
    async def cleanup(self, ctx: Context):
        """
        Cleans up bot messages in the last 100 messages.
        """
        channel = ctx.channel  # type: discord.TextChannel
        if channel.permissions_for(ctx.guild.me).manage_messages:
            count = len(await channel.purge(limit=100, check=lambda m: m.author == ctx.guild.me, before=ctx.message))
        else:
            count = 0
            async for message in channel.history(limit=100, before=ctx.message):
                if message.author == ctx.guild.me:
                    await message.delete()
                    count += 1

        m = await ctx.send(":heavy_check_mark: Cleaned up `{}` messages.".format(count))
        await asyncio.sleep(5)
        await m.delete()

setup = Misc.setup
<|endoftext|>


<commit_before>import numpy as np

# Constante universal de los gases R [=] definir las unidades #####
RGAS = 0.08314472
# Definir el significado fisicoquímico

A0, B0, C0 = 0.0017, 1.9681, -2.7238
# Definir el significado fisicoquímico

A1, B1, C1 = -2.4407, 7.4513, 12.504
# Definir el significado fisicoquímico

D = np.array([0.428363, 18.496215, 0.338426, 0.660, 789.723105, 2.512392])
<commit_msg>envelop<commit_after>import numpy as np

# Constante universal de los gases R [=] definir las unidades #####
RGAS = 0.08314472
# Definir el significado fisicoquímico

A0, B0, C0 = 0.0017, 1.9681, -2.7238
# Definir el significado fisicoquímico

A1, B1, C1 = -2.4407, 7.4513, 12.504
# Definir el significado fisicoquímico

D = np.array([0.428363, 18.496215, 0.338426, 0.660, 789.723105, 2.512392])

# definir la correlación de los valores para los paramteros A0, B0, C0 y A1, B1, C1 y D
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#   importuploads.py — Import FTP uploads to Debexpo
#
#   This file is part of debexpo - https://alioth.debian.org/projects/debexpo/
#
#   Copyright © 2011 Arno Töll <debian@toell.net>
#
#   Permission is hereby granted, free of charge, to any person
#   obtaining a copy of this software and associated documentation
#   files (the "Software"), to deal in the Software without
#   restriction, including without limitation the rights to use,
#   copy, modify, merge, publish, distribute, sublicense, and/or sell
#   copies of the Software, and to permit persons to whom the
#   Software is furnished to do so, subject to the following
#   conditions:
#
#   The above copyright notice and this permission notice shall be
#   included in all copies or substantial portions of the Software.
#
#   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
#   OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
#   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
#   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
#   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#   OTHER DEALINGS IN THE SOFTWARE.

"""
Import FTP uploads to Debexpo
"""
__author__ = 'Arno Töll'
__copyright__ = 'Copyright © 2011 Arno Töll'
__license__ = 'MIT'

from debexpo.cronjobs import BaseCronjob

import glob
import os
import os.path
import subprocess
import datetime
import shutil

import debexpo.lib.filesystem
from debexpo.lib.changes import Changes

class NotCompleteUpload(Exception): pass

class ImportUpload(BaseCronjob):
    def setup(self):
        """
        This method does nothing in this cronjob
        """
        self.stale_files = []
        self.files = debexpo.lib.filesystem.CheckFiles()
        self.log.debug("%s loaded successfully" % (__name__))

    def teardown(self):
        """
        This method does nothing in this cronjob
        """
        pass

    def invoke(self):
        """
        Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file
        """
        if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(self.config['debexpo.upload.incoming']):
            self.log.critical("debexpo.upload.incoming was not configured")
            return

        # 1) Process uploads
        for file in glob.glob( os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), '*.changes') ):
            try:
                changes = Changes(filename=file)
            except:
                self.log.error('Invalid changes file: %s' % (file))
                continue

            try:
                for filename in changes.get_files() + [ changes.get_filename(), ]:
                    source_file = os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), filename)
                    destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)

                    if not os.path.exists(source_file):
                            self.log.debug("Source file %s does not exist - putting upload on hold" % (source_file))
                            raise NotCompleteUpload;
                    if os.path.exists(destination_file):
                            self.log.debug("Do not import %s: already exists on destination path - removing file instead" % (source_file))
                            os.remove(source_file)
                            raise NotCompleteUpload;
                    shutil.move(source_file, self.config['debexpo.upload.incoming'])
            except NotCompleteUpload:
                continue

            self.log.info("Import upload: %s" % (file))
            command = [ self.config['debexpo.importer'], '-i', self.config['global_conf']['__file__'], '-c', changes.get_filename() ]
            self.log.debug("Executing: %s" % (" ".join(command)))
            proc = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (istdout, istderr) = proc.communicate()
            if proc.returncode != 0:
                self.log.critical("Importer failed to import package %s [err=%d]." % (file, proc.returncode))
                self.log.debug("Output was\n%s\n%s" % (istdout,istderr))
            for filename in changes.get_files() + [ changes.get_filename(), ]:
                destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)
                if os.path.exists(destination_file):
                    self.log.debug("Remove stale file %s - the importer probably crashed" % (destination_file))
                    os.remove(destination_file)

        # 2) Mark unprocessed files and get rid of them after some time
        pub = os.path.join(self.config['debexpo.upload.incoming'], "pub")
        filenames = [name for (name, _) in self.stale_files]
        file_to_check = []
        for file in glob.glob( os.path.join(pub, '*') ):
            if self.files.allowed_upload(file):
                self.log.debug("Incomplete upload: %s" % (file))
                if not file in filenames:
                    self.stale_files.append((file,datetime.datetime.now()))
                else:
                    file_to_check.append(file)
            else:
                if os.path.isfile(file):
                    self.log.warning("Remove unknown file: %s" % (file))
                    os.remove(file)

        new_file_list = []
        for file in file_to_check:
            for (file_known, last_check) in self.stale_files:
                if file == file_known and (datetime.datetime.now() - last_check) > datetime.timedelta(hours = 6):
                    if os.path.isfile(file):
                        self.log.warning("Remove incomplete upload: %s" % (file))
                        os.remove(file)
                        continue
                new_file_list.append((file_known, last_check))

        self.stale_files = new_file_list

cronjob = ImportUpload
schedule = datetime.timedelta(minutes = 10)
<commit_msg>Remove stale files if their mtime is older than 6 hours<commit_after># -*- coding: utf-8 -*-
#
#   importuploads.py — Import FTP uploads to Debexpo
#
#   This file is part of debexpo - https://alioth.debian.org/projects/debexpo/
#
#   Copyright © 2011 Arno Töll <debian@toell.net>
#
#   Permission is hereby granted, free of charge, to any person
#   obtaining a copy of this software and associated documentation
#   files (the "Software"), to deal in the Software without
#   restriction, including without limitation the rights to use,
#   copy, modify, merge, publish, distribute, sublicense, and/or sell
#   copies of the Software, and to permit persons to whom the
#   Software is furnished to do so, subject to the following
#   conditions:
#
#   The above copyright notice and this permission notice shall be
#   included in all copies or substantial portions of the Software.
#
#   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
#   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
#   OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
#   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
#   HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
#   WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
#   FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
#   OTHER DEALINGS IN THE SOFTWARE.

"""
Import FTP uploads to Debexpo
"""
__author__ = 'Arno Töll'
__copyright__ = 'Copyright © 2011 Arno Töll'
__license__ = 'MIT'

from debexpo.cronjobs import BaseCronjob

import glob
import os
import os.path
import subprocess
import time
import datetime
import shutil

import debexpo.lib.filesystem
from debexpo.lib.changes import Changes

class NotCompleteUpload(Exception): pass

class ImportUpload(BaseCronjob):
    def setup(self):
        """
        This method does nothing in this cronjob
        """
        self.files = debexpo.lib.filesystem.CheckFiles()
        self.log.debug("%s loaded successfully" % (__name__))

    def teardown(self):
        """
        This method does nothing in this cronjob
        """
        pass

    def invoke(self):
        """
        Loops through the debexpo.upload.incoming directory and runs the debexpo.importer for each file
        """
        if 'debexpo.upload.incoming' not in self.config or not os.path.isdir(self.config['debexpo.upload.incoming']):
            self.log.critical("debexpo.upload.incoming was not configured")
            return

        # 1) Process uploads
        for file in glob.glob( os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), '*.changes') ):
            try:
                changes = Changes(filename=file)
            except:
                self.log.error('Invalid changes file: %s' % (file))
                continue

            try:
                for filename in changes.get_files() + [ changes.get_filename(), ]:
                    source_file = os.path.join(os.path.join(self.config['debexpo.upload.incoming'], "pub"), filename)
                    destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)

                    if not os.path.exists(source_file):
                            self.log.debug("Source file %s does not exist - putting upload on hold" % (source_file))
                            raise NotCompleteUpload;
                    if os.path.exists(destination_file):
                            self.log.debug("Do not import %s: already exists on destination path - removing file instead" % (source_file))
                            os.remove(source_file)
                            raise NotCompleteUpload;
                    shutil.move(source_file, self.config['debexpo.upload.incoming'])
            except NotCompleteUpload:
                continue

            self.log.info("Import upload: %s" % (file))
            command = [ self.config['debexpo.importer'], '-i', self.config['global_conf']['__file__'], '-c', changes.get_filename() ]
            self.log.debug("Executing: %s" % (" ".join(command)))
            proc = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (istdout, istderr) = proc.communicate()
            if proc.returncode != 0:
                self.log.critical("Importer failed to import package %s [err=%d]." % (file, proc.returncode))
                self.log.debug("Output was\n%s\n%s" % (istdout,istderr))
            for filename in changes.get_files() + [ changes.get_filename(), ]:
                destination_file = os.path.join(self.config['debexpo.upload.incoming'], filename)
                if os.path.exists(destination_file):
                    self.log.debug("Remove stale file %s - the importer probably crashed" % (destination_file))
                    os.remove(destination_file)

        # 2) Mark unprocessed files and get rid of them after some time
        pub = os.path.join(self.config['debexpo.upload.incoming'], "pub")
        for file in glob.glob( os.path.join(pub, '*') ):
            if self.files.allowed_upload(file):
                self.log.debug("Incomplete upload: %s" % (file))
                last_change = time.time() - os.stat(file).st_mtime
                # the file was uploaded more than 6 hours ago
                if last_change > 6 * 60 * 60:
                    self.log.warning("Remove old file: %s (last modified %.2f hours ago)" % (file, last_change / 3600.))
                    os.remove(file)
            else:
                if os.path.isfile(file):
                    self.log.warning("Remove unknown file: %s" % (file))
                    os.remove(file)

cronjob = ImportUpload
schedule = datetime.timedelta(minutes = 10)
<|endoftext|>


<commit_before>from athenet.layers import InceptionLayer, ConvolutionalLayer, MaxPool
from athenet.algorithm.derest.network import get_derest_layer
from athenet.algorithm.derest.utils import change_order, add_tuples
from athenet.algorithm.numlike import Nplike, Interval
import theano
import theano.tensor as T

import numpy as np

def eval_tensor_on_layer(layer, tensor):
    layer.input = T.tensor4('tmp')
    evaluate = theano.function(
        [layer.input_layer.input],
        layer.output
    )
    return evaluate(tensor)

def eval_tensor_on_layers(layer1, layer2, tensor):
    layer1.input = T.tensor4()
    layer2.input_layer = layer1
    evaluate = theano.function(
        [layer1.input],
        layer2.output
    )
    return evaluate(tensor)

def testa():
    n = 3

    dummy_layer = ConvolutionalLayer((1,1,1),(n,n,1))
    input = np.ones((1, 1, n, n), dtype=theano.config.floatX)
    dummy_layer.input = input
    layer = InceptionLayer([1,2,3,4,5,6])
    layer.input_layer = dummy_layer

    print "Layer exists"
    eval_tensor_on_layers(dummy_layer, layer, input)

    print "Layer evaluated"
    print layer.output_shape
    print layer.input_shape
    derest_layer = get_derest_layer(layer)
    print "Derest layer exist"
    derest_layer.normalize = True
    act = Interval(np.zeros((1,n,n)), np.ones((1,n,n)))

    a = T.tensor3(name = "lollower", dtype=theano.config.floatX)
    b = T.tensor3(name="lolupper", dtype=theano.config.floatX)
    act_in_theory = Interval(a,b)

    derest_layer.activations = act_in_theory
    out_in_theory = derest_layer.count_activation(act_in_theory)
    print "run!"
    res = [r.eval({b: act.upper, a:act.lower})[0] for r in out_in_theory]
    print res
    shapes = [r.shape for r in res]
    print shapes

    da = T.tensor4(name="ochlower", dtype=theano.config.floatX)
    db = T.tensor4(name="ochupper", dtype=theano.config.floatX)
    print "and now test derivatives..."

    batches = 1
    input_shape = add_tuples(batches,
                             change_order(derest_layer.layer.input_shape))
    output_shape = add_tuples(batches,
                              change_order(derest_layer.layer.output_shape))
    der_in_theory = Interval(da,db)

    derest_layer.derivatives = der_in_theory
    derivatives_in_theory = derest_layer.count_derivatives(der_in_theory, input_shape)
    der_low = np.ones(output_shape)
    der_up = 2 * np.ones(output_shape)
    print output_shape

    res = derivatives_in_theory.eval({da:der_low, db:der_up, a:act.lower, b:act.upper})[0]
    print res
    print res.shape

def test_numlike():
    a = np.ones((3,4,5))
    b = 2 * a
    na = Nplike(a)
    nb = Nplike(b)
    print na.concat(nb).eval()
    ia = Interval(a, a * 3)
    ib = Interval(b, b * 3)

    print ia

    print ia.concat(ib).eval()

testa()<commit_msg>Delete unused imports<commit_after>from athenet.algorithm.derest.network import get_derest_layer
from athenet.algorithm.derest.utils import change_order, add_tuples
from athenet.algorithm.numlike import Nplike, Interval
import theano
import theano.tensor as T

import numpy as np

def eval_tensor_on_layer(layer, tensor):
    layer.input = T.tensor4('tmp')
    evaluate = theano.function(
        [layer.input_layer.input],
        layer.output
    )
    return evaluate(tensor)

def eval_tensor_on_layers(layer1, layer2, tensor):
    layer1.input = T.tensor4()
    layer2.input_layer = layer1
    evaluate = theano.function(
        [layer1.input],
        layer2.output
    )
    return evaluate(tensor)

def testa():
    n = 3

    dummy_layer = ConvolutionalLayer((1,1,1),(n,n,1))
    input = np.ones((1, 1, n, n), dtype=theano.config.floatX)
    dummy_layer.input = input
    layer = InceptionLayer([1,2,3,4,5,6])
    layer.input_layer = dummy_layer

    print "Layer exists"
    eval_tensor_on_layers(dummy_layer, layer, input)

    print "Layer evaluated"
    print layer.output_shape
    print layer.input_shape
    derest_layer = get_derest_layer(layer)
    print "Derest layer exist"
    derest_layer.normalize = True
    act = Interval(np.zeros((1,n,n)), np.ones((1,n,n)))

    a = T.tensor3(name = "lollower", dtype=theano.config.floatX)
    b = T.tensor3(name="lolupper", dtype=theano.config.floatX)
    act_in_theory = Interval(a,b)

    derest_layer.activations = act_in_theory
    out_in_theory = derest_layer.count_activation(act_in_theory)
    print "run!"
    res = [r.eval({b: act.upper, a:act.lower})[0] for r in out_in_theory]
    print res
    shapes = [r.shape for r in res]
    print shapes

    da = T.tensor4(name="ochlower", dtype=theano.config.floatX)
    db = T.tensor4(name="ochupper", dtype=theano.config.floatX)
    print "and now test derivatives..."

    batches = 1
    input_shape = add_tuples(batches,
                             change_order(derest_layer.layer.input_shape))
    output_shape = add_tuples(batches,
                              change_order(derest_layer.layer.output_shape))
    der_in_theory = Interval(da,db)

    derest_layer.derivatives = der_in_theory
    derivatives_in_theory = derest_layer.count_derivatives(der_in_theory, input_shape)
    der_low = np.ones(output_shape)
    der_up = 2 * np.ones(output_shape)
    print output_shape

    res = derivatives_in_theory.eval({da:der_low, db:der_up, a:act.lower, b:act.upper})[0]
    print res
    print res.shape

def test_numlike():
    a = np.ones((3,4,5))
    b = 2 * a
    na = Nplike(a)
    nb = Nplike(b)
    print na.concat(nb).eval()
    ia = Interval(a, a * 3)
    ib = Interval(b, b * 3)

    print ia

    print ia.concat(ib).eval()

testa()
<|endoftext|>


<commit_before>from django.contrib import admin
from metaci.build.models import Build
from metaci.build.models import BuildFlow
from metaci.build.models import FlowTask
from metaci.build.models import Rebuild

class BuildAdmin(admin.ModelAdmin):
    list_display = (
        'repo',
        'plan',
        'branch',
        'commit',
        'status',
        'time_queue',
        'time_start',
        'time_end',
    )
    list_filter = ('repo', 'plan')
    list_select_related = ('branch','repo','plan')

    raw_id_fields = ('branch', 'plan', 'repo', 'org', 'org_instance', 'current_rebuild')
admin.site.register(Build, BuildAdmin)

class BuildFlowAdmin(admin.ModelAdmin):
    list_display = (
        'build',
        'status',
        'time_queue',
        'time_start',
        'time_end',
    )
    list_filter = ('build__repo', 'build')
admin.site.register(BuildFlow, BuildFlowAdmin)

class FlowTaskAdmin(admin.ModelAdmin):
    list_display = ('id', 'build_flow', 'stepnum', 'name', 'status')
    list_filter = ('build_flow__build__repo',)
admin.site.register(FlowTask, FlowTaskAdmin)

class RebuildAdmin(admin.ModelAdmin):
    list_display = (
        'build',
        'user',
        'status',
        'time_queue',
        'time_start',
        'time_end',
    )
    list_filter = ('build__repo', 'build')
admin.site.register(Rebuild, RebuildAdmin)
<commit_msg>set raw id field for admin perf<commit_after>from django.contrib import admin
from metaci.build.models import Build
from metaci.build.models import BuildFlow
from metaci.build.models import FlowTask
from metaci.build.models import Rebuild

class BuildAdmin(admin.ModelAdmin):
    list_display = (
        'repo',
        'plan',
        'branch',
        'commit',
        'status',
        'time_queue',
        'time_start',
        'time_end',
    )
    list_filter = ('repo', 'plan')
    list_select_related = ('branch','repo','plan')

    raw_id_fields = ('branch', 'plan', 'repo', 'org', 'org_instance', 'current_rebuild')
admin.site.register(Build, BuildAdmin)

class BuildFlowAdmin(admin.ModelAdmin):
    list_display = (
        'build',
        'status',
        'time_queue',
        'time_start',
        'time_end',
    )
    list_filter = ('build__repo', 'build')
admin.site.register(BuildFlow, BuildFlowAdmin)

class FlowTaskAdmin(admin.ModelAdmin):
    list_display = ('id', 'build_flow', 'stepnum', 'name', 'status')
    list_filter = ('build_flow__build__repo',)
    raw_id_fields = ['build_flow']
admin.site.register(FlowTask, FlowTaskAdmin)

class RebuildAdmin(admin.ModelAdmin):
    list_display = (
        'build',
        'user',
        'status',
        'time_queue',
        'time_start',
        'time_end',
    )
    list_filter = ('build__repo', 'build')
admin.site.register(Rebuild, RebuildAdmin)
<|endoftext|>


<commit_before>from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.Alphabet import generic_dna

import matplotlib.pyplot as plt
import sys
import pandas as pd
import networkx as nx

"""
A small utility that produces Gibson/CPEC assembly primers to be assembled
into a circular plasmid.

Input: A single FASTA file containing nucleotide sequences in clockwise order
of assembly. The nucleotide sequence should all be the same strand.

Output: A set of named 40-mer primers required for the assembly, along with
their sequences, and the predicted PCR product size.

Assumptions:
- The region of annealing is not repeated anywhere, i.e. it is unique.
- The plasmid sequence is predicted to be stable (no homologous recombination
  possible).

Dependencies:
- biopython
- pandas
- networkx
- matplotlib
"""

class PrimerDesigner(object):

    """docstring for PrimerDesigner"""

    def __init__(self):
        super(PrimerDesigner, self).__init__()
        # The FASTA file to assemble.
        self.filename = None

        # The sequences to assemble.
        self.sequences = None

        # A NetworkX graph that stores the assembly graph.
        self.graph = nx.DiGraph()

    def read_sequences(self, filename):
        """
        Reads in the PCR products to assemble, which have been formatted as a
        FASTA file.
        """
        self.filename = filename
        self.sequences = [s for s in SeqIO.parse(filename, 'fasta')]

    def set_sequences(self, seq_records):
        """
        Takes in a list of BioPython SeqRecord objects, in the order that they
        are to be assembled, and sets the self.sequences attribute to that.

        This method exists to provide a consistent API, so that it's possible
        to set sequences by using a setter method rather than by directly
        modifying the attribute
        """
        assert isinstance(seq_records, list)

        for s in seq_records:
            assert s.id != ''
            assert s.id is not None
            assert s.id != 'None'

        self.sequences = seq_records

    def construct_graph(self):
        for i, s in enumerate(self.sequences):
            self.graph.add_node(s)
            if i > 0:
                prev_seq = self.sequences[i - 1]
                self.graph.add_edge(prev_seq, s)
            if i == len(self.sequences) - 1:
                zeroth_seq = self.sequences[0]
                self.graph.add_edge(s, zeroth_seq)

    def design_assembly_primers(self):
        """
        Given the sequences present in the graph, design primers that are
        15 n.t. overhang and 25 n.t. annealing.
        """
        for n, d in self.graph.nodes(data=True):
            current = n
            predecessor = self.graph.predecessors(n)[0]
            successor = self.graph.successors(n)[0]

            fw_primer = SeqRecord(predecessor.seq[-15:] + current.seq[0:25])
            re_primer = SeqRecord(current.seq[-25:] +
                                  successor.seq[0:15]).reverse_complement()

            self.graph.node[n]['fw_sequence'] = fw_primer
            self.graph.node[n]['re_sequence'] = re_primer
            self.graph.node[n]['fw_primer_name'] = self.filename.split(
                '.')[0] + '_' + n.id + '_fw'
            self.graph.node[n]['re_primer_name'] = self.filename.split(
                '.')[0] + '_' + n.id + '_re'

    def design_sequencing_primers(self):
        """
        For each junction (i.e. edge) in the graph, design primers that are
        positioned at -100 on the upstream part relative to the junction, and
        +100 on the downstream part relative to the junction.
        """

        for upstream, downstream, d in self.graph.edges(data=True):
            fw_primer = SeqRecord(upstream.seq[-125:-100])
            re_primer = SeqRecord(downstream.seq[100:125]).reverse_complement()

            self.graph.node[upstream]['fw_sequencing_primer'] = fw_primer
            self.graph.node[downstream]['re_sequencing_primer'] = re_primer

    def compute_pcr_protocol(self):
        """
        Returns a list of dictionaries.

        Design note: This format is really flexible, can be converted into a
        pandas dataframe later on.
        """
        assert self.filename is not None
        assert self.filename != ''

        pcr_protocol = list()
        for n, d in self.graph.nodes(data=True):
            primers = dict()

            f_id = self.filename.split('.')[0] + '_' + n.id + '_fw'
            r_id = self.filename.split('.')[0] + '_' + n.id + '_re'

            primers['fw_sequence'] = d['fw_sequence'].seq
            primers['re_sequence'] = d['re_sequence'].seq
            primers['fw_primer_name'] = d['fw_primer_name']
            primers['re_primer_name'] = d['re_primer_name']
            primers['fw_len'] = len(d['fw_sequence'])
            primers['re_len'] = len(d['re_sequence'])
            primers['template'] = n.id
            primers['pcr_length'] = len(n) + 30
            primers['phusion_extension_minutes'] = (len(n) + 30) / 1000 * 0.5
            primers['fw_sequencing_primer'] = d['fw_sequencing_primer'].seq
            primers['re_sequencing_primer'] = d['re_sequencing_primer'].seq
            pcr_protocol.append(primers)

        self.pcr_protocol = pcr_protocol

    def save_pcr_protocol(self):
        """
        Saves the PCR protocol as a CSV file.
        """

        pd.DataFrame(self.pcr_protocol).to_csv(
            "{filename}.csv".format(filename=self.filename.split('.')[0]))

# if __name__ == '__main__':

#     filename = sys.argv[1]
#     pdesigner = PrimerDesigner(filename)
#     pdesigner.design_primers()
#     pdesigner.compute_pcr_protocol()
#     pdesigner.save_pcr_protocol()
<commit_msg>added comments to set_sequences()<commit_after>from Bio import SeqIO
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio.Alphabet import generic_dna

import matplotlib.pyplot as plt
import sys
import pandas as pd
import networkx as nx

"""
A small utility that produces Gibson/CPEC assembly primers to be assembled
into a circular plasmid.

Input: A single FASTA file containing nucleotide sequences in clockwise order
of assembly. The nucleotide sequence should all be the same strand.

Output: A set of named 40-mer primers required for the assembly, along with
their sequences, and the predicted PCR product size.

Assumptions:
- The region of annealing is not repeated anywhere, i.e. it is unique.
- The plasmid sequence is predicted to be stable (no homologous recombination
  possible).

Dependencies:
- biopython
- pandas
- networkx
- matplotlib
"""

class PrimerDesigner(object):

    """docstring for PrimerDesigner"""

    def __init__(self):
        super(PrimerDesigner, self).__init__()
        # The FASTA file to assemble.
        self.filename = None

        # The sequences to assemble.
        self.sequences = None

        # A NetworkX graph that stores the assembly graph.
        self.graph = nx.DiGraph()

    def read_sequences(self, filename):
        """
        Reads in the PCR products to assemble, which have been formatted as a
        FASTA file.
        """
        self.filename = filename
        self.sequences = [s for s in SeqIO.parse(filename, 'fasta')]

    def set_sequences(self, seq_records):
        """
        Takes in a list of BioPython SeqRecord objects, in the order that they
        are to be assembled, and sets the self.sequences attribute to that.

        This method exists rather than setting the attribute directly, so as
        to perform some basic checks on the SeqRecords that are passed in.
        """
        # Make sure that seq_records is a list.
        assert isinstance(seq_records, list)

        # checks on each SeqRecord
        for s in seq_records:
            # Ensure that it is a SeqRecord
            assert isinstance(s, SeqRecord)

            # Ensure that it has an id that isn't some variant of 'None'
            assert s.id != ''
            assert s.id is not None
            assert s.id != 'None'

        self.sequences = seq_records

    def construct_graph(self):
        for i, s in enumerate(self.sequences):
            self.graph.add_node(s)
            if i > 0:
                prev_seq = self.sequences[i - 1]
                self.graph.add_edge(prev_seq, s)
            if i == len(self.sequences) - 1:
                zeroth_seq = self.sequences[0]
                self.graph.add_edge(s, zeroth_seq)

    def design_assembly_primers(self):
        """
        Given the sequences present in the graph, design primers that are
        15 n.t. overhang and 25 n.t. annealing.
        """
        for n, d in self.graph.nodes(data=True):
            current = n
            predecessor = self.graph.predecessors(n)[0]
            successor = self.graph.successors(n)[0]

            fw_primer = SeqRecord(predecessor.seq[-15:] + current.seq[0:25])
            re_primer = SeqRecord(current.seq[-25:] +
                                  successor.seq[0:15]).reverse_complement()

            self.graph.node[n]['fw_sequence'] = fw_primer
            self.graph.node[n]['re_sequence'] = re_primer
            self.graph.node[n]['fw_primer_name'] = self.filename.split(
                '.')[0] + '_' + n.id + '_fw'
            self.graph.node[n]['re_primer_name'] = self.filename.split(
                '.')[0] + '_' + n.id + '_re'

    def design_sequencing_primers(self):
        """
        For each junction (i.e. edge) in the graph, design primers that are
        positioned at -100 on the upstream part relative to the junction, and
        +100 on the downstream part relative to the junction.
        """

        for upstream, downstream, d in self.graph.edges(data=True):
            fw_primer = SeqRecord(upstream.seq[-125:-100])
            re_primer = SeqRecord(downstream.seq[100:125]).reverse_complement()

            self.graph.node[upstream]['fw_sequencing_primer'] = fw_primer
            self.graph.node[downstream]['re_sequencing_primer'] = re_primer

    def compute_pcr_protocol(self):
        """
        Returns a list of dictionaries.

        Design note: This format is really flexible, can be converted into a
        pandas dataframe later on.
        """
        assert self.filename is not None
        assert self.filename != ''

        pcr_protocol = list()
        for n, d in self.graph.nodes(data=True):
            primers = dict()

            f_id = self.filename.split('.')[0] + '_' + n.id + '_fw'
            r_id = self.filename.split('.')[0] + '_' + n.id + '_re'

            primers['fw_sequence'] = d['fw_sequence'].seq
            primers['re_sequence'] = d['re_sequence'].seq
            primers['fw_primer_name'] = d['fw_primer_name']
            primers['re_primer_name'] = d['re_primer_name']
            primers['fw_len'] = len(d['fw_sequence'])
            primers['re_len'] = len(d['re_sequence'])
            primers['template'] = n.id
            primers['pcr_length'] = len(n) + 30
            primers['phusion_extension_minutes'] = (len(n) + 30) / 1000 * 0.5
            primers['fw_sequencing_primer'] = d['fw_sequencing_primer'].seq
            primers['re_sequencing_primer'] = d['re_sequencing_primer'].seq
            pcr_protocol.append(primers)

        self.pcr_protocol = pcr_protocol

    def save_pcr_protocol(self):
        """
        Saves the PCR protocol as a CSV file.
        """

        pd.DataFrame(self.pcr_protocol).to_csv(
            "{filename}.csv".format(filename=self.filename.split('.')[0]))

# if __name__ == '__main__':

#     filename = sys.argv[1]
#     pdesigner = PrimerDesigner(filename)
#     pdesigner.design_primers()
#     pdesigner.compute_pcr_protocol()
#     pdesigner.save_pcr_protocol()
<|endoftext|>