<commit_before>"""
This module implements a simple REST API that behaves like SONATA's gatekeeper.

It is only used to support the development of SONATA's SDK tools and to demonstrate
the year 1 version of the emulator until the integration with WP4's orchestrator is done.
"""

import logging
import os
import uuid
import hashlib
import zipfile
import yaml
from docker import Client as DockerClient
from flask import Flask, request
import flask_restful as fr

logging.basicConfig()
LOG = logging.getLogger("sonata-dummy-gatekeeper")
LOG.setLevel(logging.DEBUG)
logging.getLogger("werkzeug").setLevel(logging.WARNING)

GK_STORAGE = "/tmp/son-dummy-gk/"
UPLOAD_FOLDER = os.path.join(GK_STORAGE, "uploads/")
CATALOG_FOLDER = os.path.join(GK_STORAGE, "catalog/")

# Enable Dockerfile build functionality
BUILD_DOCKERFILE = False

# flag to indicate that we run without the emulator (only the bare API for integration testing)
GK_STANDALONE_MODE = False

class Gatekeeper(object):

    def __init__(self):
        self.services = dict()
        self.dcs = dict()
        self.vnf_counter = 0  # used to generate short names for VNFs (Mininet limitation)
        LOG.info("Create SONATA dummy gatekeeper.")

    def register_service_package(self, service_uuid, service):
        """
        register new service package
        :param service_uuid
        :param service object
        """
        self.services[service_uuid] = service
        # lets perform all steps needed to onboard the service
        service.onboard()

    def get_next_vnf_name(self):
        self.vnf_counter += 1
        return "vnf%d" % self.vnf_counter

class Service(object):
    """
    This class represents a NS uploaded as a *.son package to the
    dummy gatekeeper.
    Can have multiple running instances of this service.
    """

    def __init__(self,
                 service_uuid,
                 package_file_hash,
                 package_file_path):
        self.uuid = service_uuid
        self.package_file_hash = package_file_hash
        self.package_file_path = package_file_path
        self.package_content_path = os.path.join(CATALOG_FOLDER, "services/%s" % self.uuid)
        self.manifest = None
        self.nsd = None
        self.vnfds = dict()
        self.local_docker_files = dict()
        self.remote_docker_image_urls = dict()
        self.instances = dict()

    def onboard(self):
        """
        Do all steps to prepare this service to be instantiated
        :return:
        """
        # 1. extract the contents of the package and store them in our catalog
        self._unpack_service_package()
        # 2. read in all descriptor files
        self._load_package_descriptor()
        self._load_nsd()
        self._load_vnfd()
        # 3. prepare container images (e.g. download or build Dockerfile)
        if BUILD_DOCKERFILE:
            self._load_docker_files()
            self._build_images_from_dockerfiles()
        else:
            self._load_docker_urls()
            self._pull_predefined_dockerimages()
        LOG.info("On-boarded service: %r" % self.manifest.get("package_name"))

    def start_service(self):
        """
        This methods creates and starts a new service instance.
        It computes placements, iterates over all VNFDs, and starts
        each VNFD as a Docker container in the data center selected
        by the placement algorithm.
        :return:
        """
        LOG.info("Starting service %r" % self.uuid)
        # 1. each service instance gets a new uuid to identify it
        instance_uuid = str(uuid.uuid4())
        # build a instances dict (a bit like a NSR :))
        self.instances[instance_uuid] = dict()
        self.instances[instance_uuid]["vnf_instances"] = list()
        # 2. compute placement of this service instance (adds DC names to VNFDs)
        if not GK_STANDALONE_MODE:
            self._calculate_placement(FirstDcPlacement)
        # iterate over all vnfds that we have to start
        for vnfd in self.vnfds.itervalues():
            vnfi = None
            if not GK_STANDALONE_MODE:
                vnfi = self._start_vnfd(vnfd)
            self.instances[instance_uuid]["vnf_instances"].append(vnfi)
        LOG.info("Service started. Instance id: %r" % instance_uuid)
        return instance_uuid

    def _start_vnfd(self, vnfd):
        """
        Start a single VNFD of this service
        :param vnfd: vnfd descriptor dict
        :return:
        """
        # iterate over all deployment units within each VNFDs
        for u in vnfd.get("virtual_deployment_units"):
            # 1. get the name of the docker image to start and the assigned DC
            docker_name = vnfd.get("name")
            target_dc = vnfd.get("dc")
            # 2. perform some checks to ensure we can start the container
            assert(docker_name is not None)
            assert(target_dc is not None)
            if not self._check_docker_image_exists(docker_name):
                raise Exception("Docker image %r not found. Abort." % docker_name)
            # 3. do the dc.startCompute(name="foobar") call to run the container
            # TODO consider flavors, and other annotations
            vnfi = target_dc.startCompute(GK.get_next_vnf_name(), image=docker_name, flavor_name="small")
            # 6. store references to the compute objects in self.instances
            return vnfi

    def _unpack_service_package(self):
        """
        unzip *.son file and store contents in CATALOG_FOLDER/services/<service_uuid>/
        """
        LOG.info("Unzipping: %r" % self.package_file_path)
        with zipfile.ZipFile(self.package_file_path, "r") as z:
            z.extractall(self.package_content_path)

    def _load_package_descriptor(self):
        """
        Load the main package descriptor YAML and keep it as dict.
        :return:
        """
        self.manifest = load_yaml(
            os.path.join(
                self.package_content_path, "META-INF/MANIFEST.MF"))

    def _load_nsd(self):
        """
        Load the entry NSD YAML and keep it as dict.
        :return:
        """
        if "entry_service_template" in self.manifest:
            nsd_path = os.path.join(
                self.package_content_path,
                make_relative_path(self.manifest.get("entry_service_template")))
            self.nsd = load_yaml(nsd_path)
            LOG.debug("Loaded NSD: %r" % self.nsd.get("name"))

    def _load_vnfd(self):
        """
        Load all VNFD YAML files referenced in MANIFEST.MF and keep them in dict.
        :return:
        """
        if "package_content" in self.manifest:
            for pc in self.manifest.get("package_content"):
                if pc.get("content-type") == "application/sonata.function_descriptor":
                    vnfd_path = os.path.join(
                        self.package_content_path,
                        make_relative_path(pc.get("name")))
                    vnfd = load_yaml(vnfd_path)
                    self.vnfds[vnfd.get("name")] = vnfd
                    LOG.debug("Loaded VNFD: %r" % vnfd.get("name"))

    def _load_docker_files(self):
        """
        Get all paths to Dockerfiles from VNFDs and store them in dict.
        :return:
        """
        for k, v in self.vnfds.iteritems():
            for vu in v.get("virtual_deployment_units"):
                if vu.get("vm_image_format") == "docker":
                    vm_image = vu.get("vm_image")
                    docker_path = os.path.join(
                        self.package_content_path,
                        make_relative_path(vm_image))
                    self.local_docker_files[k] = docker_path
                    LOG.debug("Found Dockerfile: %r" % docker_path)

    def _load_docker_urls(self):
        """
        Get all URLs to pre-build docker images in some repo.
        :return:
        """
        for k, v in self.vnfds.iteritems():
            for vu in v.get("virtual_deployment_units"):
                if vu.get("vm_image_format") == "docker":
                    self.remote_docker_image_urls[k] = vu.get("vm_image")
                    LOG.debug("Found Docker image URL: %r" %  self.remote_docker_image_urls[k])

    def _build_images_from_dockerfiles(self):
        """
        Build Docker images for each local Dockerfile found in the package: self.local_docker_files
        """
        if GK_STANDALONE_MODE:
            return  # do not build anything in standalone mode
        dc = DockerClient()
        LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files))
        for k, v in self.local_docker_files.iteritems():
            for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False):
                LOG.debug("DOCKER BUILD: %s" % line)
            LOG.info("Docker image created: %s" % k)

    def _pull_predefined_dockerimages(self):
        """
        If the package contains URLs to pre-build Docker images, we download them with this method.
        """
        # TODO implement this
        pass

    def _check_docker_image_exists(self, image_name):
        """
        Query the docker service and check if the given image exists
        :param image_name: name of the docker image
        :return:
        """
        return len(DockerClient().images(image_name)) > 0

    def _calculate_placement(self, algorithm):
        """
        Do placement by adding the a field "dc" to
        each VNFD that points to one of our
        data center objects known to the gatekeeper.
        """
        assert(len(self.vnfds) > 0)
        assert(len(GK.dcs) > 0)
        # instantiate algorithm an place
        p = algorithm()
        p.place(self.nsd, self.vnfds, GK.dcs)
        LOG.info("Using placement algorithm: %r" % p.__class__.__name__)
        # lets print the placement result
        for name, vnfd in self.vnfds.iteritems():
            LOG.info("Placed VNF %r on DC %r" % (name, str(vnfd.get("dc"))))

"""
Some (simple) placement algorithms
"""

class FirstDcPlacement(object):
    """
    Placement: Always use one and the same data center from the GK.dcs dict.
    """
    def place(self, nsd, vnfds, dcs):
        for name, vnfd in vnfds.iteritems():
            vnfd["dc"] = list(dcs.itervalues())[0]

"""
Resource definitions and API endpoints
"""

class Packages(fr.Resource):

    def post(self):
        """
        Upload a *.son service package to the dummy gatekeeper.

        We expect request with a *.son file and store it in UPLOAD_FOLDER
        :return: UUID
        """
        try:
            # get file contents
            print(request.files)
            # lets search for the package in the request
            if "package" in request.files:
                son_file = request.files["package"]
            # elif "file" in request.files:
            #     son_file = request.files["file"]
            else:
                return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed. file not found."}, 500
            # generate a uuid to reference this package
            service_uuid = str(uuid.uuid4())
            file_hash = hashlib.sha1(str(son_file)).hexdigest()
            # ensure that upload folder exists
            ensure_dir(UPLOAD_FOLDER)
            upload_path = os.path.join(UPLOAD_FOLDER, "%s.son" % service_uuid)
            # store *.son file to disk
            son_file.save(upload_path)
            size = os.path.getsize(upload_path)
            # create a service object and register it
            s = Service(service_uuid, file_hash, upload_path)
            GK.register_service_package(service_uuid, s)
            # generate the JSON result
            return {"service_uuid": service_uuid, "size": size, "sha1": file_hash, "error": None}
        except Exception as ex:
            LOG.exception("Service package upload failed:")
            return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed"}, 500

    def get(self):
        """
        Return a list of UUID's of uploaded service packages.
        :return: dict/list
        """
        return {"service_uuid_list": list(GK.services.iterkeys())}

class Instantiations(fr.Resource):

    def post(self):
        """
        Instantiate a service specified by its UUID.
        Will return a new UUID to identify the running service instance.
        :return: UUID
        """
        # try to extract the service uuid from the request
        json_data = request.get_json(force=True)
        service_uuid = json_data.get("service_uuid")

        # lets be a bit fuzzy here to make testing easier
        if service_uuid is None and len(GK.services) > 0:
            # if we don't get a service uuid, we simple start the first service in the list
            service_uuid = list(GK.services.iterkeys())[0]

        if service_uuid in GK.services:
            # ok, we have a service uuid, lets start the service
            service_instance_uuid = GK.services.get(service_uuid).start_service()
            return {"service_instance_uuid": service_instance_uuid}
        return "Service not found", 404

    def get(self):
        """
        Returns a list of UUIDs containing all running services.
        :return: dict / list
        """
        return {"service_instance_list": [
            list(s.instances.iterkeys()) for s in GK.services.itervalues()]}

# create a single, global GK object
GK = Gatekeeper()
# setup Flask
app = Flask(__name__)
app.config['MAX_CONTENT_LENGTH'] = 512 * 1024 * 1024  # 512 MB max upload
api = fr.Api(app)
# define endpoints
api.add_resource(Packages, '/packages')
api.add_resource(Instantiations, '/instantiations')

def start_rest_api(host, port, datacenters=dict()):
    GK.dcs = datacenters
    # start the Flask server (not the best performance but ok for our use case)
    app.run(host=host,
            port=port,
            debug=True,
            use_reloader=False  # this is needed to run Flask in a non-main thread
            )

def ensure_dir(name):
    if not os.path.exists(name):
        os.makedirs(name)

def load_yaml(path):
    with open(path, "r") as f:
        try:
            r = yaml.load(f)
        except yaml.YAMLError as exc:
            LOG.exception("YAML parse error")
            r = dict()
    return r

def make_relative_path(path):
    if path.startswith("file://"):
        path = path.replace("file://", "", 1)
    if path.startswith("/"):
        path = path.replace("/", "", 1)
    return path

if __name__ == '__main__':
    """
    Lets allow to run the API in standalone mode.
    """
    GK_STANDALONE_MODE = True
    logging.getLogger("werkzeug").setLevel(logging.INFO)
    start_rest_api("0.0.0.0", 8000)

<commit_msg>first version of docker pull functionality<commit_after>"""
This module implements a simple REST API that behaves like SONATA's gatekeeper.

It is only used to support the development of SONATA's SDK tools and to demonstrate
the year 1 version of the emulator until the integration with WP4's orchestrator is done.
"""

import logging
import os
import uuid
import hashlib
import zipfile
import yaml
from docker import Client as DockerClient
from flask import Flask, request
import flask_restful as fr

logging.basicConfig()
LOG = logging.getLogger("sonata-dummy-gatekeeper")
LOG.setLevel(logging.DEBUG)
logging.getLogger("werkzeug").setLevel(logging.WARNING)

GK_STORAGE = "/tmp/son-dummy-gk/"
UPLOAD_FOLDER = os.path.join(GK_STORAGE, "uploads/")
CATALOG_FOLDER = os.path.join(GK_STORAGE, "catalog/")

# Enable Dockerfile build functionality
BUILD_DOCKERFILE = False

# flag to indicate that we run without the emulator (only the bare API for integration testing)
GK_STANDALONE_MODE = False

class Gatekeeper(object):

    def __init__(self):
        self.services = dict()
        self.dcs = dict()
        self.vnf_counter = 0  # used to generate short names for VNFs (Mininet limitation)
        LOG.info("Create SONATA dummy gatekeeper.")

    def register_service_package(self, service_uuid, service):
        """
        register new service package
        :param service_uuid
        :param service object
        """
        self.services[service_uuid] = service
        # lets perform all steps needed to onboard the service
        service.onboard()

    def get_next_vnf_name(self):
        self.vnf_counter += 1
        return "vnf%d" % self.vnf_counter

class Service(object):
    """
    This class represents a NS uploaded as a *.son package to the
    dummy gatekeeper.
    Can have multiple running instances of this service.
    """

    def __init__(self,
                 service_uuid,
                 package_file_hash,
                 package_file_path):
        self.uuid = service_uuid
        self.package_file_hash = package_file_hash
        self.package_file_path = package_file_path
        self.package_content_path = os.path.join(CATALOG_FOLDER, "services/%s" % self.uuid)
        self.manifest = None
        self.nsd = None
        self.vnfds = dict()
        self.local_docker_files = dict()
        self.remote_docker_image_urls = dict()
        self.instances = dict()

    def onboard(self):
        """
        Do all steps to prepare this service to be instantiated
        :return:
        """
        # 1. extract the contents of the package and store them in our catalog
        self._unpack_service_package()
        # 2. read in all descriptor files
        self._load_package_descriptor()
        self._load_nsd()
        self._load_vnfd()
        # 3. prepare container images (e.g. download or build Dockerfile)
        if BUILD_DOCKERFILE:
            self._load_docker_files()
            self._build_images_from_dockerfiles()
        else:
            self._load_docker_urls()
            self._pull_predefined_dockerimages()
        LOG.info("On-boarded service: %r" % self.manifest.get("package_name"))

    def start_service(self):
        """
        This methods creates and starts a new service instance.
        It computes placements, iterates over all VNFDs, and starts
        each VNFD as a Docker container in the data center selected
        by the placement algorithm.
        :return:
        """
        LOG.info("Starting service %r" % self.uuid)
        # 1. each service instance gets a new uuid to identify it
        instance_uuid = str(uuid.uuid4())
        # build a instances dict (a bit like a NSR :))
        self.instances[instance_uuid] = dict()
        self.instances[instance_uuid]["vnf_instances"] = list()
        # 2. compute placement of this service instance (adds DC names to VNFDs)
        if not GK_STANDALONE_MODE:
            self._calculate_placement(FirstDcPlacement)
        # iterate over all vnfds that we have to start
        for vnfd in self.vnfds.itervalues():
            vnfi = None
            if not GK_STANDALONE_MODE:
                vnfi = self._start_vnfd(vnfd)
            self.instances[instance_uuid]["vnf_instances"].append(vnfi)
        LOG.info("Service started. Instance id: %r" % instance_uuid)
        return instance_uuid

    def _start_vnfd(self, vnfd):
        """
        Start a single VNFD of this service
        :param vnfd: vnfd descriptor dict
        :return:
        """
        # iterate over all deployment units within each VNFDs
        for u in vnfd.get("virtual_deployment_units"):
            # 1. get the name of the docker image to start and the assigned DC
            docker_name = vnfd.get("name")
            target_dc = vnfd.get("dc")
            # 2. perform some checks to ensure we can start the container
            assert(docker_name is not None)
            assert(target_dc is not None)
            if not self._check_docker_image_exists(docker_name):
                raise Exception("Docker image %r not found. Abort." % docker_name)
            # 3. do the dc.startCompute(name="foobar") call to run the container
            # TODO consider flavors, and other annotations
            vnfi = target_dc.startCompute(GK.get_next_vnf_name(), image=docker_name, flavor_name="small")
            # 6. store references to the compute objects in self.instances
            return vnfi

    def _unpack_service_package(self):
        """
        unzip *.son file and store contents in CATALOG_FOLDER/services/<service_uuid>/
        """
        LOG.info("Unzipping: %r" % self.package_file_path)
        with zipfile.ZipFile(self.package_file_path, "r") as z:
            z.extractall(self.package_content_path)

    def _load_package_descriptor(self):
        """
        Load the main package descriptor YAML and keep it as dict.
        :return:
        """
        self.manifest = load_yaml(
            os.path.join(
                self.package_content_path, "META-INF/MANIFEST.MF"))

    def _load_nsd(self):
        """
        Load the entry NSD YAML and keep it as dict.
        :return:
        """
        if "entry_service_template" in self.manifest:
            nsd_path = os.path.join(
                self.package_content_path,
                make_relative_path(self.manifest.get("entry_service_template")))
            self.nsd = load_yaml(nsd_path)
            LOG.debug("Loaded NSD: %r" % self.nsd.get("name"))

    def _load_vnfd(self):
        """
        Load all VNFD YAML files referenced in MANIFEST.MF and keep them in dict.
        :return:
        """
        if "package_content" in self.manifest:
            for pc in self.manifest.get("package_content"):
                if pc.get("content-type") == "application/sonata.function_descriptor":
                    vnfd_path = os.path.join(
                        self.package_content_path,
                        make_relative_path(pc.get("name")))
                    vnfd = load_yaml(vnfd_path)
                    self.vnfds[vnfd.get("name")] = vnfd
                    LOG.debug("Loaded VNFD: %r" % vnfd.get("name"))

    def _load_docker_files(self):
        """
        Get all paths to Dockerfiles from VNFDs and store them in dict.
        :return:
        """
        for k, v in self.vnfds.iteritems():
            for vu in v.get("virtual_deployment_units"):
                if vu.get("vm_image_format") == "docker":
                    vm_image = vu.get("vm_image")
                    docker_path = os.path.join(
                        self.package_content_path,
                        make_relative_path(vm_image))
                    self.local_docker_files[k] = docker_path
                    LOG.debug("Found Dockerfile: %r" % docker_path)

    def _load_docker_urls(self):
        """
        Get all URLs to pre-build docker images in some repo.
        :return:
        """
        for k, v in self.vnfds.iteritems():
            for vu in v.get("virtual_deployment_units"):
                if vu.get("vm_image_format") == "docker":
                    url = vu.get("vm_image")
                    if url is not None:
                        url = url.replace("http://", "")
                        self.remote_docker_image_urls[k] = url
                        LOG.debug("Found Docker image URL: %r" % self.remote_docker_image_urls[k])

    def _build_images_from_dockerfiles(self):
        """
        Build Docker images for each local Dockerfile found in the package: self.local_docker_files
        """
        if GK_STANDALONE_MODE:
            return  # do not build anything in standalone mode
        dc = DockerClient()
        LOG.info("Building %d Docker images (this may take several minutes) ..." % len(self.local_docker_files))
        for k, v in self.local_docker_files.iteritems():
            for line in dc.build(path=v.replace("Dockerfile", ""), tag=k, rm=False, nocache=False):
                LOG.debug("DOCKER BUILD: %s" % line)
            LOG.info("Docker image created: %s" % k)

    def _pull_predefined_dockerimages(self):
        """
        If the package contains URLs to pre-build Docker images, we download them with this method.
        """
        # TODO implement this
        dc = DockerClient()
        for url in self.remote_docker_image_urls.itervalues():
            LOG.info("Pulling image: %r" % url)
            dc.pull(url,
                    insecure_registry=True)

    def _check_docker_image_exists(self, image_name):
        """
        Query the docker service and check if the given image exists
        :param image_name: name of the docker image
        :return:
        """
        return len(DockerClient().images(image_name)) > 0

    def _calculate_placement(self, algorithm):
        """
        Do placement by adding the a field "dc" to
        each VNFD that points to one of our
        data center objects known to the gatekeeper.
        """
        assert(len(self.vnfds) > 0)
        assert(len(GK.dcs) > 0)
        # instantiate algorithm an place
        p = algorithm()
        p.place(self.nsd, self.vnfds, GK.dcs)
        LOG.info("Using placement algorithm: %r" % p.__class__.__name__)
        # lets print the placement result
        for name, vnfd in self.vnfds.iteritems():
            LOG.info("Placed VNF %r on DC %r" % (name, str(vnfd.get("dc"))))

"""
Some (simple) placement algorithms
"""

class FirstDcPlacement(object):
    """
    Placement: Always use one and the same data center from the GK.dcs dict.
    """
    def place(self, nsd, vnfds, dcs):
        for name, vnfd in vnfds.iteritems():
            vnfd["dc"] = list(dcs.itervalues())[0]

"""
Resource definitions and API endpoints
"""

class Packages(fr.Resource):

    def post(self):
        """
        Upload a *.son service package to the dummy gatekeeper.

        We expect request with a *.son file and store it in UPLOAD_FOLDER
        :return: UUID
        """
        try:
            # get file contents
            print(request.files)
            # lets search for the package in the request
            if "package" in request.files:
                son_file = request.files["package"]
            # elif "file" in request.files:
            #     son_file = request.files["file"]
            else:
                return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed. file not found."}, 500
            # generate a uuid to reference this package
            service_uuid = str(uuid.uuid4())
            file_hash = hashlib.sha1(str(son_file)).hexdigest()
            # ensure that upload folder exists
            ensure_dir(UPLOAD_FOLDER)
            upload_path = os.path.join(UPLOAD_FOLDER, "%s.son" % service_uuid)
            # store *.son file to disk
            son_file.save(upload_path)
            size = os.path.getsize(upload_path)
            # create a service object and register it
            s = Service(service_uuid, file_hash, upload_path)
            GK.register_service_package(service_uuid, s)
            # generate the JSON result
            return {"service_uuid": service_uuid, "size": size, "sha1": file_hash, "error": None}
        except Exception as ex:
            LOG.exception("Service package upload failed:")
            return {"service_uuid": None, "size": 0, "sha1": None, "error": "upload failed"}, 500

    def get(self):
        """
        Return a list of UUID's of uploaded service packages.
        :return: dict/list
        """
        return {"service_uuid_list": list(GK.services.iterkeys())}

class Instantiations(fr.Resource):

    def post(self):
        """
        Instantiate a service specified by its UUID.
        Will return a new UUID to identify the running service instance.
        :return: UUID
        """
        # try to extract the service uuid from the request
        json_data = request.get_json(force=True)
        service_uuid = json_data.get("service_uuid")

        # lets be a bit fuzzy here to make testing easier
        if service_uuid is None and len(GK.services) > 0:
            # if we don't get a service uuid, we simple start the first service in the list
            service_uuid = list(GK.services.iterkeys())[0]

        if service_uuid in GK.services:
            # ok, we have a service uuid, lets start the service
            service_instance_uuid = GK.services.get(service_uuid).start_service()
            return {"service_instance_uuid": service_instance_uuid}
        return "Service not found", 404

    def get(self):
        """
        Returns a list of UUIDs containing all running services.
        :return: dict / list
        """
        return {"service_instance_list": [
            list(s.instances.iterkeys()) for s in GK.services.itervalues()]}

# create a single, global GK object
GK = Gatekeeper()
# setup Flask
app = Flask(__name__)
app.config['MAX_CONTENT_LENGTH'] = 512 * 1024 * 1024  # 512 MB max upload
api = fr.Api(app)
# define endpoints
api.add_resource(Packages, '/packages')
api.add_resource(Instantiations, '/instantiations')

def start_rest_api(host, port, datacenters=dict()):
    GK.dcs = datacenters
    # start the Flask server (not the best performance but ok for our use case)
    app.run(host=host,
            port=port,
            debug=True,
            use_reloader=False  # this is needed to run Flask in a non-main thread
            )

def ensure_dir(name):
    if not os.path.exists(name):
        os.makedirs(name)

def load_yaml(path):
    with open(path, "r") as f:
        try:
            r = yaml.load(f)
        except yaml.YAMLError as exc:
            LOG.exception("YAML parse error")
            r = dict()
    return r

def make_relative_path(path):
    if path.startswith("file://"):
        path = path.replace("file://", "", 1)
    if path.startswith("/"):
        path = path.replace("/", "", 1)
    return path

if __name__ == '__main__':
    """
    Lets allow to run the API in standalone mode.
    """
    GK_STANDALONE_MODE = True
    logging.getLogger("werkzeug").setLevel(logging.INFO)
    start_rest_api("0.0.0.0", 8000)

<|endoftext|>


<commit_before>"""Base classes and mixins.
"""

class _NotSet(object):
    """Represents an unset value."""
    def __repr__(self):  # pragma: no cover
        return 'NotSet'

#: Singleton to indicate that a keyword argument was not provided.
NotSet = _NotSet()

class Assertion(object):
    """Base class for assertions.

    If `value` **is not** provided, then assertion isn't executed. This style
    of usage is used in conjuction with :class:`.expect`.

    If `value` **is** provided, then assertion is executed immediately. This
    style of usage is used when making assertions using only the class and not
    an assertion runner like :class:`.expect`.

    Keyword Arguments:
        msg (str, optional): Override assert message to use when performing
            assertion.
    """
    #: Default format string used for assert message.
    reason = ''

    #: Operation to perform to determine whether `value` is valid. **This must
    #: be set in subclass**.
    op = None

    def __init__(self, value=NotSet, **opts):
        self.set_options(opts)

        if value is not NotSet:
            self(value, **opts)

    def set_options(self, opts):
        # Optional method that sets options as class instance variables for
        # use when calling operation.
        pass

    def format_msg(self, *args, **kargs):
        """Return formatted assert message. This is used to generate the assert
        message during :meth:`__call__`. If no ``msg`` keyword argument is
        provided, then :attr:`reason` will be used as the format string. By
        default, passed in ``args`` and ``kargs`` along with the classes
        ``__dict__`` dictionary are given to the format string. In all cases,
        ``arg[0]`` will be the `value` that is being validated.
        """
        reason = kargs.pop('msg', None) or self.reason
        kargs.update(self.__dict__)
        return reason.format(*args, **kargs)

    def compare(self, value):  # pragma: no cover
        # pylint: disable=not-callable
        return self.op(value)

    def __repr__(self):  # pragma: no cover
        return '<{0}>'.format(self)

    def __str__(self):  # pragma: no cover
        return '{0}()'.format(self.__class__.__name__)

    def __call__(self, *args, **opts):
        """Execute validation.

        Keyword Arguments:
            msg (str, optional): Override assert message to use when performing
                assertion.

        Returns:
            bool: ``True`` if comparison passes, otherwise, an
                ``AssertionError`` is raised.

        Raises:
            AssertionError: If comparison returns ``False``.
        """
        fmt_kargs = {'msg': opts.pop('msg', None)}
        fmt_kargs.update(opts)

        assert self.compare(*args, **opts), self.format_msg(*args, **fmt_kargs)
        return True

class Comparator(Assertion):
    """Base class for assertions that compare two values."""
    def __init__(self, comparable, value=NotSet, **opts):
        if value is not NotSet:
            # Swap variables since the prescence of both inputs indicates we
            # are immediately executing validation.
            value, comparable = comparable, value

        # Whether we are validating now or later, set comparable on class since
        # self.compare() expects comparable to be an instance variable.
        self.comparable = comparable

        super(Comparator, self).__init__(value, **opts)

    def compare(self, value):
        # pylint: disable=not-callable
        return self.op(value, self.comparable)

class Negate(object):
    """Mixin class that negates the results of :meth:`compare` from the parent
    class.
    """
    def compare(self, *args, **opts):
        try:
            return not super(Negate, self).compare(*args, **opts)
        except AssertionError:  # pragma: no cover
            return True

def is_assertion(obj):
    """Return whether `obj` is either an instance or subclass of
    :class:`Assertion`.
    """
    is_instance = isinstance(obj, Assertion)

    try:
        is_subclass = issubclass(obj, Assertion)
    except TypeError:
        # Happens if `obj` isn't a class.
        is_subclass = False

    return is_instance or is_subclass
<commit_msg>Refactor of is_assertion.<commit_after>"""Base classes and mixins.
"""

class _NotSet(object):
    """Represents an unset value."""
    def __repr__(self):  # pragma: no cover
        return 'NotSet'

#: Singleton to indicate that a keyword argument was not provided.
NotSet = _NotSet()

class Assertion(object):
    """Base class for assertions.

    If `value` **is not** provided, then assertion isn't executed. This style
    of usage is used in conjuction with :class:`.expect`.

    If `value` **is** provided, then assertion is executed immediately. This
    style of usage is used when making assertions using only the class and not
    an assertion runner like :class:`.expect`.

    Keyword Arguments:
        msg (str, optional): Override assert message to use when performing
            assertion.
    """
    #: Default format string used for assert message.
    reason = ''

    #: Operation to perform to determine whether `value` is valid. **This must
    #: be set in subclass**.
    op = None

    def __init__(self, value=NotSet, **opts):
        self.set_options(opts)

        if value is not NotSet:
            self(value, **opts)

    def set_options(self, opts):
        # Optional method that sets options as class instance variables for
        # use when calling operation.
        pass

    def format_msg(self, *args, **kargs):
        """Return formatted assert message. This is used to generate the assert
        message during :meth:`__call__`. If no ``msg`` keyword argument is
        provided, then :attr:`reason` will be used as the format string. By
        default, passed in ``args`` and ``kargs`` along with the classes
        ``__dict__`` dictionary are given to the format string. In all cases,
        ``arg[0]`` will be the `value` that is being validated.
        """
        reason = kargs.pop('msg', None) or self.reason
        kargs.update(self.__dict__)
        return reason.format(*args, **kargs)

    def compare(self, value):  # pragma: no cover
        # pylint: disable=not-callable
        return self.op(value)

    def __repr__(self):  # pragma: no cover
        return '<{0}>'.format(self)

    def __str__(self):  # pragma: no cover
        return '{0}()'.format(self.__class__.__name__)

    def __call__(self, *args, **opts):
        """Execute validation.

        Keyword Arguments:
            msg (str, optional): Override assert message to use when performing
                assertion.

        Returns:
            bool: ``True`` if comparison passes, otherwise, an
                ``AssertionError`` is raised.

        Raises:
            AssertionError: If comparison returns ``False``.
        """
        fmt_kargs = {'msg': opts.pop('msg', None)}
        fmt_kargs.update(opts)

        assert self.compare(*args, **opts), self.format_msg(*args, **fmt_kargs)
        return True

class Comparator(Assertion):
    """Base class for assertions that compare two values."""
    def __init__(self, comparable, value=NotSet, **opts):
        if value is not NotSet:
            # Swap variables since the prescence of both inputs indicates we
            # are immediately executing validation.
            value, comparable = comparable, value

        # Whether we are validating now or later, set comparable on class since
        # self.compare() expects comparable to be an instance variable.
        self.comparable = comparable

        super(Comparator, self).__init__(value, **opts)

    def compare(self, value):
        # pylint: disable=not-callable
        return self.op(value, self.comparable)

class Negate(object):
    """Mixin class that negates the results of :meth:`compare` from the parent
    class.
    """
    def compare(self, *args, **opts):
        try:
            return not super(Negate, self).compare(*args, **opts)
        except AssertionError:  # pragma: no cover
            return True

def is_assertion(obj):
    """Return whether `obj` is either an instance or subclass of
    :class:`Assertion`.
    """
    try:
        return isinstance(obj, Assertion) or issubclass(obj, Assertion)
    except TypeError:
        # Happens if `obj` isn't a class.
        return False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

try:
    # noinspection PyUnresolvedReferences,PyShadowingBuiltins
    str = unicode
except NameError:
    # noinspection PyShadowingBuiltins,PyUnboundLocalVariable
    str = str

try:
    from json import JSONDecodeError
except ImportError:
    JSONDecodeError = ValueError

from itertools import chain

import requests
from requests.exceptions import RequestException
import weechat

LICENSE = "MIT"

NAME = "pybullet"
VERSION = 0.1
AUTHOR = "Kent Ross"
__doc__ = (
    "{0} {1}: Push smart notifications to pushbullet. Authored by {2}"
    .format(NAME, VERSION, AUTHOR)
)

BULLET_URL = "https://api.pushbullet.com/v2/"

session = requests.session()
session.headers['User-Agent'] = "{0}/{1}".format(NAME, VERSION)

# Registration #

weechat.register(
    NAME,
    AUTHOR,
    str(VERSION),
    LICENSE,
    __doc__,  # description
    "",  # shutdown_function
    ""  # charset, default utf-8
)

# Configuration #

# options (default, type, description)
config = {
    'api_secret': (
        "",
        weechat.config_string,
        "PushBullet access token"
    ),

    'notification_title': (
        "weechat",
        weechat.config_string,
        "Title for notifications sent"
    ),

    'only_when_away': (
        False,
        weechat.config_boolean,
        "Only send notifications when away (default: off)"
    ),

    'highlights': (
        True,
        weechat.config_boolean,
        "Send notifications for highlights (default: on)"
    ),

    'privmsg': (
        True,
        weechat.config_boolean,
        "Send notifications for private messages (default: on)"
    ),

    'displayed_messages': (
        3,
        weechat.config_integer,
        "Number of messages for which to display the full text (default: 3)"
    ),

    'count_limit': (
        10,
        weechat.config_integer,
        "More than this many messages will be reported as 'many' instead of a "
        "specific number of messages (default: 10)"
    ),

    'short_buffer_name': (
        False,
        weechat.config_boolean,
        "Use the short name of the buffer rather than the long one "
        "(default: off)"
    ),

    'debug': (
        True,
        weechat.config_boolean,
        "Print debug info while the app is running"
    ),
}

# functions to convert configs read from the application
config_types = {}

def debug(text):
    if True:  # config['debug']:
        weechat.prnt("", "{0}: {1}".format(NAME, text))

def config_as_str(value):
    """Convert config defaults to strings for weechat"""
    if isinstance(value, bool):
        return "on" if value else "off"
    else:
        return str(value)

def init_config():
    """Perform initial configuration of the application settings"""
    for option, (default_value, config_type, description) in config.items():
        x = weechat.config_get_plugin(option)
        debug("Raw value for {0} is {1} (type {2})".format(option, repr(x), type(x)))
        # set config type
        config_types[option] = config_type
        # set descriptions for options
        weechat.config_set_desc_plugin(option, description)
        # setdefault the script's options from weechat
        if not weechat.config_is_set_plugin(option):
            weechat.config_set_plugin(option, config_as_str(default_value))
            config[option] = default_value
            debug('Option "{0}" was not set, is now {1}'.format(option, repr(default_value)))
        else:
            config[option] = config_type(weechat.config_get_plugin(option))
            debug('Option "{0}" set to {1}'.format(option, repr(config[option])))

init_config()

def config_cb(data, option, value):
    """Called when a config option is changed."""
    if data != "config":
        debug("Got wrong data in config_cb: {0}".format(data))
        return weechat.WEECHAT_RC_ERROR
    if option in config:
        config[option] = config_types[option](value)
        debug('Option "{0}" set to "{1}" as {2}'.format(option, value, repr(config[option])))
    else:
        debug('Option "{0}" does not seem to be in config')
    return weechat.WEECHAT_RC_OK

# Notification functions #

class Notification(object):
    """Class to track notifications for a particular buffer"""

    by_buffer = {}

    def __init__(self):
        self.buffer_show = ""
        self.messages = []
        self.count = 0
        self.iden = None

    def add_message(self, show_buffer_name, message):
        """Add a message to this notification and update the push"""
        self.buffer_show = show_buffer_name

        if not config['api_secret']:
            debug("no access token set, aborting")
            return

        changed = False

        self.check_dismissal()

        # remember message if it will be displayed
        to_display = config['displayed_messages']
        if to_display != 0 and to_display < len(self.messages):
            self.messages.append(message)
            changed = True

        # update count of messages
        if type(self.count) is int:
            self.count += 1
            changed = True
            if self.count > config['count_limit']:
                self.count = "Many"

        if changed:
            # Notification's content has changed, update the push
            self.repost()

    def json(self):
        """Create the notification's push data for its current state"""
        return {
            'type': "note",
            'title': config['notification_title'],
            'body': "\n".join(chain(
                ["{1}[{0}]".format(
                    self.buffer_show,
                    "{0} messages from ".format(self.count) if self.count > 1 else ""
                )],
                self.messages,
                ["..."] if self.count > len(self.messages) else ()
            ))
        }

    def check_dismissal(self):
        """Check if this notification's push was dismissed and reset it if so."""
        if not self.iden:
            return
        try:
            res = session.get(BULLET_URL + "pushes/{0}".format(self.iden))
        except RequestException as ex:
            debug("Bad error while getting push info: {0}".format(ex))
            return

        if res.status_code == 200:
            try:
                if res.json()['dismissed']:
                    # reset self
                    debug("Push for {0} was dismissed".format(self.buffer_show))
                    self.messages.clear()
                    self.count = 0
                    self.iden = None
            except (JSONDecodeError, KeyError) as ex:
                debug("Error while reading push info: {0}".format(ex))
        else:
            debug("Error while getting push info: status {0}".format(res.status_code))

    def repost(self):
        """Delete this notification's current push, then post a new one."""
        debug("Reposting for {0} from iden {1}".format(self.buffer_show, self.iden))
        if self.iden:
            res = session.delete(BULLET_URL + "pushes/{0}".format(self.iden))
            if res.status_code not in (200, 404):
                debug(
                    "Failed to delete pushes/{0} with status code {1}"
                    .format(self.iden, res.status_code)
                )
            else:
                self.iden = None

        # Now post the new push
        try:
            res = session.post(
                BULLET_URL + "pushes",
                headers={'Access-Token': config['api_secret']},
                json=self.json()
            )
        except RequestException as ex:
            debug("Bad error while posting push: {0}".format(ex))
            return
        if res.status_code == 200:
            try:
                self.iden = res.json()['iden']
                debug("Got new iden {0}".format(self.iden))
            except (JSONDecodeError, KeyError) as ex:
                debug("Error reading push creation response: {0}".format(ex))
        else:
            debug("Error posting push: status {0}".format(res.status_code))

def dispatch_notification(buffer_name, show_buffer_name, message_text):
    note = Notification.by_buffer.get(buffer_name)
    if not note:
        note = Notification()
        Notification.by_buffer[buffer_name] = note
    note.add_message(show_buffer_name, message_text)

# Core callbacks #

# inspector doesn't like unused parameters
# noinspection PyUnusedLocal
def print_cb(data, buffer_ptr, date, tag_count, is_displayed, is_highlight, prefix, message):
    """
    Called from weechat when something is printed.

    This is only hooked to relevant prints (private and highlight) so it is generally
    not necessary to check for the former.
    """
    if data != "print":
        debug("Got wrong data in print_cb: {0}".format(data))
        return weechat.WEECHAT_RC_ERROR

    buffer_name = weechat.buffer_get_string(buffer_ptr, 'full_name')

    # away rules: cancel
    if config['only_when_away'] and not weechat.buffer_get_string(buffer_ptr, 'localvar_away'):
        debug("Message for {0} ignored due to away status".format(buffer_name))

    # highlight or private message
    elif (
        (is_highlight and config['highlights']) or
        (config['privmsg'])
    ):
        if config['short_buffer_name']:
            show_buffer_name = weechat.buffer_get_string(buffer_ptr, 'short_name')
        else:
            show_buffer_name = buffer_name

        debug("Dispatching notification for {0}".format(buffer_name))

        # send the notification
        dispatch_notification(
            buffer_name, show_buffer_name,
            "<{0}> {1}".format(prefix, message)
        )

    else:
        debug("Not dispatching notification for {0}".format(buffer_name))

    return weechat.WEECHAT_RC_OK

# Register callbacks #

weechat.hook_print(
    "",                                         # buffer (blank: any buffer)
    "notify_private,notify_highlight",          # print tags to catch
    "",                                         # message must contain this string
    1,                                          # 1 if strip colors from message
    'print_cb',                                 # name of callback function
    "print"                                     # data given to callback function
)
weechat.hook_config(
    "plugins.var.python.{0}.*".format(NAME),    # filter for configs to watch
    'config_cb',                                # name of callback function
    "config"                                    # data given to callback function
)

weechat.prnt("", "{0}: loaded and running. Debug is {1}".format(NAME, config_as_str(config['debug'])))
<commit_msg>different option parsing<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

try:
    # noinspection PyUnresolvedReferences,PyShadowingBuiltins
    str = unicode
except NameError:
    # noinspection PyShadowingBuiltins,PyUnboundLocalVariable
    str = str

try:
    from json import JSONDecodeError
except ImportError:
    JSONDecodeError = ValueError

from itertools import chain

import requests
from requests.exceptions import RequestException
import weechat

LICENSE = "MIT"

NAME = "pybullet"
VERSION = 0.1
AUTHOR = "Kent Ross"
__doc__ = (
    "{0} {1}: Push smart notifications to pushbullet. Authored by {2}"
    .format(NAME, VERSION, AUTHOR)
)

BULLET_URL = "https://api.pushbullet.com/v2/"

session = requests.session()
session.headers['User-Agent'] = "{0}/{1}".format(NAME, VERSION)

# Registration #

weechat.register(
    NAME,
    AUTHOR,
    str(VERSION),
    LICENSE,
    __doc__,  # description
    "",  # shutdown_function
    ""  # charset, default utf-8
)

# Configuration #

def option_string(value):
    return value

def option_boolean(value):
    return value.lower() in ("on", "yes", "y", "true", "t", "1")

def option_integer(value):
    try:
        return int(value)
    except ValueError:
        return 0

# options (default, type, description)
config = {
    'api_secret': (
        "",
        option_string,
        "PushBullet access token"
    ),

    'notification_title': (
        "weechat",
        option_string,
        "Title for notifications sent"
    ),

    'only_when_away': (
        False,
        option_boolean,
        "Only send notifications when away (default: off)"
    ),

    'highlights': (
        True,
        option_boolean,
        "Send notifications for highlights (default: on)"
    ),

    'privmsg': (
        True,
        option_boolean,
        "Send notifications for private messages (default: on)"
    ),

    'displayed_messages': (
        3,
        option_integer,
        "Number of messages for which to display the full text (default: 3)"
    ),

    'count_limit': (
        10,
        option_integer,
        "More than this many messages will be reported as 'many' instead of a "
        "specific number of messages (default: 10)"
    ),

    'short_buffer_name': (
        False,
        option_boolean,
        "Use the short name of the buffer rather than the long one "
        "(default: off)"
    ),

    'debug': (
        True,
        option_boolean,
        "Print debug info while the app is running"
    ),
}

# functions to convert configs read from the application
config_types = {}

def debug(text):
    if True:  # config['debug']:
        weechat.prnt("", "{0}: {1}".format(NAME, text))

def config_as_str(value):
    """Convert config defaults to strings for weechat"""
    if isinstance(value, bool):
        return "on" if value else "off"
    else:
        return str(value)

def init_config():
    """Perform initial configuration of the application settings"""
    for option, (default_value, config_type, description) in config.items():
        x = weechat.config_get_plugin(option)
        debug("Raw value for {0} is {1} (type {2})".format(option, repr(x), type(x)))
        # set config type
        config_types[option] = config_type
        # set descriptions for options
        weechat.config_set_desc_plugin(option, description)
        # setdefault the script's options from weechat
        if not weechat.config_is_set_plugin(option):
            weechat.config_set_plugin(option, config_as_str(default_value))
            config[option] = default_value
            debug('Option "{0}" was not set, is now {1}'.format(option, repr(default_value)))
        else:
            config[option] = config_type(weechat.config_get_plugin(option))
            debug('Option "{0}" set to {1}'.format(option, repr(config[option])))

init_config()

def config_cb(data, option, value):
    """Called when a config option is changed."""
    debug("Config callback: {0} {1} {2}".format(data, option, value))
    if data != "config":
        debug("Got wrong data in config_cb: {0}".format(data))
        return weechat.WEECHAT_RC_ERROR
    if option in config:
        config[option] = config_types[option](value)
        debug('Option "{0}" set to "{1}" as {2}'.format(option, value, repr(config[option])))
    else:
        debug('Option "{0}" does not seem to be in config')
    return weechat.WEECHAT_RC_OK

# Notification functions #

class Notification(object):
    """Class to track notifications for a particular buffer"""

    by_buffer = {}

    def __init__(self):
        self.buffer_show = ""
        self.messages = []
        self.count = 0
        self.iden = None

    def add_message(self, show_buffer_name, message):
        """Add a message to this notification and update the push"""
        self.buffer_show = show_buffer_name

        if not config['api_secret']:
            debug("no access token set, aborting")
            return

        changed = False

        self.check_dismissal()

        # remember message if it will be displayed
        to_display = config['displayed_messages']
        if to_display != 0 and to_display < len(self.messages):
            self.messages.append(message)
            changed = True

        # update count of messages
        if type(self.count) is int:
            self.count += 1
            changed = True
            if self.count > config['count_limit']:
                self.count = "Many"

        if changed:
            # Notification's content has changed, update the push
            self.repost()

    def json(self):
        """Create the notification's push data for its current state"""
        return {
            'type': "note",
            'title': config['notification_title'],
            'body': "\n".join(chain(
                ["{1}[{0}]".format(
                    self.buffer_show,
                    "{0} messages from ".format(self.count) if self.count > 1 else ""
                )],
                self.messages,
                ["..."] if self.count > len(self.messages) else ()
            ))
        }

    def check_dismissal(self):
        """Check if this notification's push was dismissed and reset it if so."""
        if not self.iden:
            return
        try:
            res = session.get(BULLET_URL + "pushes/{0}".format(self.iden))
        except RequestException as ex:
            debug("Bad error while getting push info: {0}".format(ex))
            return

        if res.status_code == 200:
            try:
                if res.json()['dismissed']:
                    # reset self
                    debug("Push for {0} was dismissed".format(self.buffer_show))
                    self.messages.clear()
                    self.count = 0
                    self.iden = None
            except (JSONDecodeError, KeyError) as ex:
                debug("Error while reading push info: {0}".format(ex))
        else:
            debug("Error while getting push info: status {0}".format(res.status_code))

    def repost(self):
        """Delete this notification's current push, then post a new one."""
        debug("Reposting for {0} from iden {1}".format(self.buffer_show, self.iden))
        if self.iden:
            res = session.delete(BULLET_URL + "pushes/{0}".format(self.iden))
            if res.status_code not in (200, 404):
                debug(
                    "Failed to delete pushes/{0} with status code {1}"
                    .format(self.iden, res.status_code)
                )
            else:
                self.iden = None

        # Now post the new push
        try:
            res = session.post(
                BULLET_URL + "pushes",
                headers={'Access-Token': config['api_secret']},
                json=self.json()
            )
        except RequestException as ex:
            debug("Bad error while posting push: {0}".format(ex))
            return
        if res.status_code == 200:
            try:
                self.iden = res.json()['iden']
                debug("Got new iden {0}".format(self.iden))
            except (JSONDecodeError, KeyError) as ex:
                debug("Error reading push creation response: {0}".format(ex))
        else:
            debug("Error posting push: status {0}".format(res.status_code))

def dispatch_notification(buffer_name, show_buffer_name, message_text):
    note = Notification.by_buffer.get(buffer_name)
    if not note:
        note = Notification()
        Notification.by_buffer[buffer_name] = note
    note.add_message(show_buffer_name, message_text)

# Core callbacks #

# inspector doesn't like unused parameters
# noinspection PyUnusedLocal
def print_cb(data, buffer_ptr, date, tag_count, is_displayed, is_highlight, prefix, message):
    """
    Called from weechat when something is printed.

    This is only hooked to relevant prints (private and highlight) so it is generally
    not necessary to check for the former.
    """
    if data != "print":
        debug("Got wrong data in print_cb: {0}".format(data))
        return weechat.WEECHAT_RC_ERROR

    buffer_name = weechat.buffer_get_string(buffer_ptr, 'full_name')

    # away rules: cancel
    if config['only_when_away'] and not weechat.buffer_get_string(buffer_ptr, 'localvar_away'):
        debug("Message for {0} ignored due to away status".format(buffer_name))

    # highlight or private message
    elif (
        (is_highlight and config['highlights']) or
        (config['privmsg'])
    ):
        if config['short_buffer_name']:
            show_buffer_name = weechat.buffer_get_string(buffer_ptr, 'short_name')
        else:
            show_buffer_name = buffer_name

        debug("Dispatching notification for {0}".format(buffer_name))

        # send the notification
        dispatch_notification(
            buffer_name, show_buffer_name,
            "<{0}> {1}".format(prefix, message)
        )

    else:
        debug("Not dispatching notification for {0}".format(buffer_name))

    return weechat.WEECHAT_RC_OK

# Register callbacks #

weechat.hook_print(
    "",                                         # buffer (blank: any buffer)
    "notify_private,notify_highlight",          # print tags to catch
    "",                                         # message must contain this string
    1,                                          # 1 if strip colors from message
    'print_cb',                                 # name of callback function
    "print"                                     # data given to callback function
)
weechat.hook_config(
    "plugins.var.python.{0}.*".format(NAME),    # filter for configs to watch
    'config_cb',                                # name of callback function
    "config"                                    # data given to callback function
)

weechat.prnt("", "{0}: loaded and running. Debug is {1}".format(NAME, config_as_str(config['debug'])))
<|endoftext|>


<commit_before>from pprint import pprint

from django.shortcuts import render
from django.urls import reverse
from django.views import View

from fo2.connections import db_cursor_so

import utils.functions.strings
from utils.views import group_rowspan, totalize_grouped_data

import produto.queries

from estoque import forms, queries

class ReferenciaDeposito(View):

    def __init__(self, *args, **kwargs):
        super(ReferenciaDeposito).__init__(*args, **kwargs)
        self.Form_class = forms.ReferenciasEstoqueForm
        self.template_name = 'estoque/referencia_deposito.html'
        self.context = {
            'titulo': 'Em depósito',
            'str_depositos': self.str_depositos(),
        }

    def mount_context(self, request, cursor, deposito, modelo):
        context = {
            'deposito': deposito,
            'modelo': modelo,
        }
        try:
            imodelo = int(modelo)
        except Exception:
            imodelo = None
        if imodelo is not None:
            anterior = None
            posterior = None
            get_prox = False
            lista = produto.queries.busca_modelo(cursor)
            for row in lista:
                item = row['modelo']
                if get_prox:
                    posterior = item
                    break
                else:
                    if imodelo == item:
                        get_prox = True
                    else:
                        anterior = item
            context.update({
                'anterior': anterior,
                'posterior': posterior,
            })

        data = queries.referencia_deposito(cursor, modelo, deposito=deposito)
        if len(data) == 0:
            context.update({'erro': 'Nada selecionado'})
            return context

        for row in data:
            row['ref|TARGET'] = '_blank'
            row['ref|LINK'] = reverse(
                'estoque:mostra_estoque__get', args=[
                    row['dep'], row['ref']])

        group = ['dep']
        tot_conf = {
            'group': group,
            'sum': ['estoque', 'falta', 'soma'],
            'count': [],
            'descr': {'ref': 'Totais:'},
        }
        if deposito == '-':
            tot_conf.update({
                'global_sum': ['estoque', 'falta', 'soma'],
                'global_descr': {'ref': 'Totais gerais:'},
            })
        totalize_grouped_data(data, tot_conf)
        group_rowspan(data, group)

        context.update({
            'headers': ['Depósito', 'Referência', 'Estoque positivo',
                        'Estoque negativo', 'Soma'],
            'fields': ['dep', 'ref', 'estoque',
                       'falta', 'soma'],
            'group': group,
            'data': data,
            'style': {
                3: 'text-align: right;',
                4: 'text-align: right;',
                5: 'text-align: right;',
            },
        })

        return context

    def str_depositos(seft):
        texto = utils.functions.strings.join(
            (', ', ' e '), (101, 102, 122, 231))
        return texto

    def get(self, request, *args, **kwargs):
        cursor = db_cursor_so(request)
        form = self.Form_class(cursor=cursor)
        self.context['form'] = form
        return render(request, self.template_name, self.context)

    def post(self, request, *args, **kwargs):
        cursor = db_cursor_so(request)
        form = self.Form_class(request.POST, cursor=cursor)
        if form.is_valid():
            deposito = form.cleaned_data['deposito']
            modelo = form.cleaned_data['modelo']
            self.context.update(self.mount_context(
                request, cursor, deposito, modelo))
        self.context['form'] = form
        return render(request, self.template_name, self.context)
<commit_msg>label depósitos<commit_after>from pprint import pprint

from django.shortcuts import render
from django.urls import reverse
from django.views import View

from fo2.connections import db_cursor_so

import utils.functions.strings
from utils.views import group_rowspan, totalize_grouped_data

import produto.queries

from estoque import forms, queries

class ReferenciaDeposito(View):

    def __init__(self, *args, **kwargs):
        super(ReferenciaDeposito).__init__(*args, **kwargs)
        self.Form_class = forms.ReferenciasEstoqueForm
        self.template_name = 'estoque/referencia_deposito.html'
        self.context = {
            'titulo': 'Em depósito',
            'str_depositos': self.str_depositos(),
        }

    def mount_context(self, request, cursor, deposito, modelo):
        context = {
            'deposito': deposito,
            'modelo': modelo,
        }
        try:
            imodelo = int(modelo)
        except Exception:
            imodelo = None
        if imodelo is not None:
            anterior = None
            posterior = None
            get_prox = False
            lista = produto.queries.busca_modelo(cursor)
            for row in lista:
                item = row['modelo']
                if get_prox:
                    posterior = item
                    break
                else:
                    if imodelo == item:
                        get_prox = True
                    else:
                        anterior = item
            context.update({
                'anterior': anterior,
                'posterior': posterior,
            })

        data = queries.referencia_deposito(cursor, modelo, deposito=deposito)
        if len(data) == 0:
            context.update({'erro': 'Nada selecionado'})
            return context

        for row in data:
            row['ref|TARGET'] = '_blank'
            row['ref|LINK'] = reverse(
                'estoque:mostra_estoque__get', args=[
                    row['dep'], row['ref']])

        group = ['dep']
        tot_conf = {
            'group': group,
            'sum': ['estoque', 'falta', 'soma'],
            'count': [],
            'descr': {'ref': 'Totais:'},
        }
        if deposito == '-':
            tot_conf.update({
                'global_sum': ['estoque', 'falta', 'soma'],
                'global_descr': {'ref': 'Totais gerais:'},
            })
        totalize_grouped_data(data, tot_conf)
        group_rowspan(data, group)

        context.update({
            'headers': ['Depósito', 'Referência', 'Estoque positivo',
                        'Estoque negativo', 'Soma'],
            'fields': ['dep', 'ref', 'estoque',
                       'falta', 'soma'],
            'group': group,
            'data': data,
            'style': {
                3: 'text-align: right;',
                4: 'text-align: right;',
                5: 'text-align: right;',
            },
        })

        return context

    def str_depositos(seft):
        texto = utils.functions.strings.join(
            (', ', ' e '), (101, 102, 103, 122, 231))
        return texto

    def get(self, request, *args, **kwargs):
        cursor = db_cursor_so(request)
        form = self.Form_class(cursor=cursor)
        self.context['form'] = form
        return render(request, self.template_name, self.context)

    def post(self, request, *args, **kwargs):
        cursor = db_cursor_so(request)
        form = self.Form_class(request.POST, cursor=cursor)
        if form.is_valid():
            deposito = form.cleaned_data['deposito']
            modelo = form.cleaned_data['modelo']
            self.context.update(self.mount_context(
                request, cursor, deposito, modelo))
        self.context['form'] = form
        return render(request, self.template_name, self.context)
<|endoftext|>


<commit_before>import json

from django.http import HttpResponseRedirect, HttpResponse

from fusionbox.core.serializers.json import FusionboxJSONEncoder

class HttpResponseSeeOther(HttpResponseRedirect):
    status_code = 303

class JsonResponse(HttpResponse):
    """
    Takes a jsonable object and returns a response with its encoded value. Also
    sets the Content-Type to json.

    Usage::

        def aview(request):
            return JsonResponse({'foo': 1})
    """

    def __init__(self, context, *args, **kwargs):
        content = json.dumps(context, default=FusionboxJSONEncoder)
        super(JsonResponse, self).__init__(content, *args, **kwargs)
        self['Content-Type'] = 'application/json'
<commit_msg>fix import<commit_after>import json

from django.http import HttpResponseRedirect, HttpResponse

from fusionbox.core.serializers import FusionboxJSONEncoder

class HttpResponseSeeOther(HttpResponseRedirect):
    status_code = 303

class JsonResponse(HttpResponse):
    """
    Takes a jsonable object and returns a response with its encoded value. Also
    sets the Content-Type to json.

    Usage::

        def aview(request):
            return JsonResponse({'foo': 1})
    """

    def __init__(self, context, *args, **kwargs):
        content = json.dumps(context, default=FusionboxJSONEncoder)
        super(JsonResponse, self).__init__(content, *args, **kwargs)
        self['Content-Type'] = 'application/json'
<|endoftext|>


<commit_before>representations = ['csv', 'json', 'primitive', 'tsv', 'txt', 'yaml']

for r in representations:
    m = __import__('thingstance.representations.'+r)
<commit_msg>Representations loads modules<commit_after>representations = {}

for r in ['csv', 'json', 'primitive', 'tsv', 'txt', 'yaml']:
    representations[r] = __import__('thingstance.representations.'+r)
<|endoftext|>


<commit_before># Copyright 2014, 2015 Red Hat
#
# Authors: Chris Dent <chdent@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""See gabbi.driver and gabbbi.case."""

__version__ = '0.5.0'
<commit_msg>release 0.5.1<commit_after># Copyright 2014, 2015 Red Hat
#
# Authors: Chris Dent <chdent@redhat.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""See gabbi.driver and gabbbi.case."""

__version__ = '0.5.1'
<|endoftext|>


<commit_before>import numpy as np
from math import log
from ..activation.util import sigmoid,sigmoidDerivative
from ..preprocess.util import *
from ..losses import LOSS
from ..optimizers.SGD import *
class brain:
	def __init__(self,ep=10e-4):
		self.e=ep
		self.layers=[]
		self.wpl=[]#weights per layer
		self.bpl=[]
	def fit(self,X,Y,layers=None,neurons=None,one_hot=False):
		""" Fits the training data:
		Parameters:
			X=training features(array like object)
			Y=training labels(array like object)
			layers=number of hidden layers(default=None)
			neurons=number of neurons for each layer (list),default=None
			one_hot=one hot vectors of the target values(training labels)
		"""
		self.class_set=set([_[0] for _ in Y if isinstance(_,(list,np.ndarray))])
		if not self.class_set:self.class_set=set(Y)
		X,Y=checkfit(X,Y)
		self.feat=X
		self.m=len(X)
		if one_hot:
			Y=one_hot_encoding(Y)
			self.target=Y
		else:self.target=Y
		if layers != None:
			if neurons==None:
				raise ValueError("Did not provide neurons to the hidden layers")
			if not isinstance(neurons,list):
				raise ValueError("parameter neurons should be of class list but given %s"%(type(neurons)))
			if not isinstance(layers,int):
				raise ValueError(" parameter layers should be of class int but given %s"%(type(layers)))
			self.totl=layers
			self.layers.append(X)
			for i in range(self.totl):
				l=np.full([neurons[i],],0.0)
				self.layers.append(l)
				if i==self.totl-1:
					out_l=np.full([len(list(self.class_set)),],0.0)
					self.layers.append(out_l)
			#for j in range(1,len(self.layers)):
			#	w=np.random.random_sample((len(self.layers[j]),len(self.layers[j-1])+1))
			for j in range(1,len(neurons)):
				w=np.random.random_sample((neurons[j],neurons[j-1]+1))
				self.wpl.append(w)
			for k in range(1,len(neurons)):
				b=np.random.random_sample((1,neurons[k]))
				self.bpl.append(b)
	def train(self,optimizer="SGD",epoch=30):
		self.opt=SGD(self.feat,self.target,epoch,self.e)
		self.f_w,self.f_b=self.opt.optimize(self.wpl,self.bpl)

<commit_msg>cleaned<commit_after>import numpy as np
from math import log
from ..activation.util import sigmoid,sigmoidDerivative
from ..preprocess.util import *
from ..losses import LOSS
from ..optimizers.SGD import *
class brain:
	def __init__(self,ep=10e-4):
		self.e=ep
		self.layers=[]
		self.wpl=[]#weights per layer
		self.bpl=[]
	def fit(self,X,Y,layers=None,neurons=None,one_hot=False):
		""" Fits the training data:
		Parameters:
			X=training features(array like object)
			Y=training labels(array like object)
			layers=number of hidden layers(default=None)
			neurons=number of neurons for each layer (list),default=None
			one_hot=one hot vectors of the target values(training labels)
		"""
		self.class_set=set([_[0] for _ in Y if isinstance(_,(list,np.ndarray))])
		if not self.class_set:self.class_set=set(Y)
		X,Y=checkfit(X,Y)
		self.feat=X
		self.m=len(X)
		if one_hot:
			Y=one_hot_encoding(Y)
			self.target=Y
		else:self.target=Y
		if layers != None:
			if neurons==None:
				raise ValueError("Did not provide neurons to the hidden layers")
			if not isinstance(neurons,list):
				raise ValueError("parameter neurons should be of class list but given %s"%(type(neurons)))
			if not isinstance(layers,int):
				raise ValueError(" parameter layers should be of class int but given %s"%(type(layers)))
			self.totl=layers
			self.layers.append(X)
			for i in range(self.totl):
				l=np.full([neurons[i],],0.0)
				self.layers.append(l)
				if i==self.totl-1:
					out_l=np.full([len(list(self.class_set)),],0.0)
					self.layers.append(out_l)
			#for j in range(1,len(self.layers)):
			#	w=np.random.random_sample((len(self.layers[j]),len(self.layers[j-1])+1))
			for j in range(1,len(neurons)):
				w=np.random.random_sample((neurons[j],neurons[j-1]+1))
				self.wpl.append(w)
			for k in range(1,len(neurons)):
				b=np.random.random_sample((1,neurons[k]))
				self.bpl.append(b)
	def train(self,optimizer="SGD",epoch=30):
		self.opt=SGD(self.feat,self.target,epoch,self.e)
		self.f_w,self.f_b=self.opt.optimize(np.asarray(self.wpl),np.asarray(self.bpl))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import numpy as np
from sklearn.metrics import f1_score
from sklearn.metrics import roc_auc_score, f1_score
import xgboost as xgb
from hyperopt import hp, fmin, tpe, STATUS_OK, Trials
from sklearn.model_selection import StratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Imputer
from data_load import load_data, load_data_tgt

# Load data
data_dir = '/home/ilya/code/ml4vs/data/LMC_SC20__corrected_list_of_variables/normalized'
file_1 = 'vast_lightcurve_statistics_normalized_variables_only.log'
file_0 = 'vast_lightcurve_statistics_normalized_constant_only.log'
file_0 = os.path.join(data_dir, file_0)
file_1 = os.path.join(data_dir, file_1)
names = ['Magnitude', 'clipped_sigma', 'meaningless_1', 'meaningless_2',
         'star_ID', 'weighted_sigma', 'skew', 'kurt', 'I', 'J', 'K', 'L',
         'Npts', 'MAD', 'lag1', 'RoMS', 'rCh2', 'Isgn', 'Vp2p', 'Jclp', 'Lclp',
         'Jtim', 'Ltim', 'CSSD', 'Ex', 'inv_eta', 'E_A', 'S_B', 'NXS', 'IQR']
names_to_delete = ['Magnitude', 'meaningless_1', 'meaningless_2', 'star_ID',
                   'Npts', 'CSSD']

X, y, df, features_names, delta = load_data([file_0, file_1], names, names_to_delete)
target = 'variable'
predictors = list(df)
predictors.remove(target)
dtrain = df

kfold = StratifiedKFold(n_splits=4, shuffle=True, random_state=123)

def xg_f1(y, t):
    t = t.get_label()
    # Binaryzing your output
    y_bin = [1. if y_cont > 0.5 else 0. for y_cont in y]
    return 'f1', 1-f1_score(t, y_bin)

def objective(space):
    clf = xgb.XGBClassifier(n_estimators=10000, learning_rate=space['lr'],
                            max_depth=space['max_depth'],
                            min_child_weight=space['min_child_weight'],
                            subsample=space['subsample'],
                            colsample_bytree=space['colsample_bytree'],
                            colsample_bylevel=space['colsample_bylevel'],
                            gamma=space['gamma'],
                            scale_pos_weight=space['scale_pos_weight'])
                            # scale_pos_weight=space['scale_pos_weight'])

    # Try using pipeline
    estimators = list()
    estimators.append(('imputer', Imputer(missing_values='NaN', strategy='median',
                                          axis=0, verbose=2)))
    # estimators.append(('scaler', StandardScaler()))
    estimators.append(('clf', clf))
    pipeline = Pipeline(estimators)

    f1_scores = list()
    best_n = ""
    for train_indx, test_indx in kfold.split(dtrain[predictors].index,
                                             dtrain['variable']):
        train = dtrain.iloc[train_indx]
        valid = dtrain.iloc[test_indx]

        valid_ = valid[predictors]
        train_ = train[predictors]
        for name, transform in pipeline.steps[:-1]:
            transform.fit(train_)
            valid_ = transform.transform(valid_)
            train_ = transform.transform(train_)

        eval_set = [(train_, train['variable']),
                    (valid_, valid['variable'])]

        pipeline.fit(train[predictors], train['variable'],
                     # clf__eval_set=eval_set, clf__eval_metric="auc",
                     clf__eval_set=eval_set, clf__eval_metric=xg_f1,
                     clf__early_stopping_rounds=50)
        best_f1_score = 1-clf.best_score

        # pred = pipeline.predict_proba(valid[predictors])[:, 1]
        # auc = roc_auc_score(valid['variable'], pred)
        # scores.append(auc)
        f1_scores.append(best_f1_score)
        best_n = best_n + " " + str(clf.best_ntree_limit)
    print "SCORE:", np.mean(f1_scores)

    return{'loss': 1-np.mean(f1_scores), 'status': STATUS_OK ,
           'attachments': {'best_n': best_n}}

space ={
    'max_depth': hp.choice("x_max_depth", np.arange(5, 12, 1, dtype=int)),
    'min_child_weight': hp.quniform('x_min_child', 1, 20, 2),
    'subsample': hp.quniform('x_subsample', 0.5, 1, 0.05),
    'colsample_bytree': hp.quniform('x_csbtree', 0.5, 1, 0.05),
    'colsample_bylevel': hp.quniform('x_csblevel', 0.5, 1, 0.05),
    'gamma': hp.quniform('x_gamma', 0.0, 1, 0.05),
    'scale_pos_weight': hp.quniform('x_spweight', 1, 303, 50),
    'lr': hp.quniform('lr', 0.001, 0.5, 0.025)
    # 'lr': hp.loguniform('lr', -7, -1)
}

trials = Trials()
best = fmin(fn=objective,
            space=space,
            algo=tpe.suggest,
            max_evals=300,
            trials=trials)

import hyperopt
print hyperopt.space_eval(space, best)

best_pars = hyperopt.space_eval(space, best)
best_n = trials.attachments['ATTACH::{}::best_n'.format(trials.best_trial['tid'])]
best_n = max([int(n) for n in best_n.strip().split(' ')])

clf = xgb.XGBClassifier(n_estimators=1000,
                        learning_rate=0.03,
                        max_depth=best_pars['max_depth'],
                        min_child_weight=best_pars['min_child_weight'],
                        subsample=best_pars['subsample'],
                        colsample_bytree=best_pars['colsample_bytree'],
                        colsample_bylevel=best_pars['colsample_bylevel'],
                        gamma=best_pars['gamma'],
                        scale_pos_weight=50.)

estimators = list()
estimators.append(('imputer', Imputer(missing_values='NaN', strategy='median',
                                      axis=0, verbose=2)))
estimators.append(('clf', clf))
pipeline = Pipeline(estimators)

# Fit classifier with best hyperparameters on whole data set
pipeline.fit(dtrain[predictors], dtrain['variable'])

# Load blind test data
file_tgt = 'LMC_SC19_PSF_Pgood98__vast_lightcurve_statistics_normalized.log'
file_tgt = os.path.join(data_dir, file_tgt)
X_tgt, feature_names, df, df_orig = load_data_tgt(file_tgt, names, names_to_delete,
                                                  delta)

y_probs = pipeline.predict_proba(df[predictors])[:, 1]
idx = y_probs > 0.5
idx_ = y_probs < 0.5
gb_no = list(df_orig['star_ID'][idx_])
print("Found {} variables".format(np.count_nonzero(idx)))

with open('gb_results.txt', 'w') as fo:
    for line in list(df_orig['star_ID'][idx]):
        fo.write(line + '\n')

# Check F1
with open('clean_list_of_new_variables.txt', 'r') as fo:
    news = fo.readlines()
news = [line.strip().split(' ')[1] for line in news]
news = set(news)

with open('gb_results.txt', 'r') as fo:
    gb = fo.readlines()
gb = [line.strip().split('_')[4].split('.')[0] for line in gb]
gb = set(gb)

print "Among new vars found {}".format(len(news.intersection(gb)))

with open('candidates_50perc_threshold.txt', 'r') as fo:
    c50 = fo.readlines()
c50 = [line.strip("\", ', \", \n, }, {") for line in c50]

with open('variables_not_in_catalogs.txt', 'r') as fo:
    not_in_cat = fo.readlines()
nic = [line.strip().split(' ')[1] for line in not_in_cat]

# Catalogue variables
cat_vars = set(c50).difference(set(nic))
# Non-catalogue variable
noncat_vars = set([line.strip().split(' ')[1] for line in not_in_cat if 'CST' not in line])

# All variables
all_vars = news.union(cat_vars).union(noncat_vars)
gb_no = set([line.strip().split('_')[4].split('.')[0] for line in gb_no])

found_bad = '181193' in gb
print "Found known variable : ", found_bad

FN = len(gb_no.intersection(all_vars))
TP = len(all_vars.intersection(gb))
TN = len(gb_no) - FN
FP = len(gb) - TP
recall = float(TP) / (TP + FN)
precision = float(TP) / (TP + FP)
F1 = 2 * precision * recall / (precision + recall)
print "precision: {}".format(precision)
print "recall: {}".format(recall)
print "F1: {}".format(F1)
print "TN={}, FP={}".format(TN, FP)
print "FN={}, TP={}".format(FN, TP)

<commit_msg>monitoring F1_val for ES, minimizing F1<commit_after># -*- coding: utf-8 -*-
import os
import numpy as np
from sklearn.metrics import confusion_matrix
from sklearn.metrics import f1_score
from sklearn.metrics import roc_auc_score, f1_score
import xgboost as xgb
from hyperopt import hp, fmin, tpe, STATUS_OK, Trials
from sklearn.model_selection import StratifiedKFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Imputer
from data_load import load_data, load_data_tgt

# Load data
data_dir = '/home/ilya/code/ml4vs/data/LMC_SC20__corrected_list_of_variables/raw_index_values'
file_1 = 'vast_lightcurve_statistics_variables_only.log'
file_0 = 'vast_lightcurve_statistics_constant_only.log'
file_0 = os.path.join(data_dir, file_0)
file_1 = os.path.join(data_dir, file_1)
names = ['Magnitude', 'clipped_sigma', 'meaningless_1', 'meaningless_2',
         'star_ID', 'weighted_sigma', 'skew', 'kurt', 'I', 'J', 'K', 'L',
         'Npts', 'MAD', 'lag1', 'RoMS', 'rCh2', 'Isgn', 'Vp2p', 'Jclp', 'Lclp',
         'Jtim', 'Ltim', 'CSSD', 'Ex', 'inv_eta', 'E_A', 'S_B', 'NXS', 'IQR']
# names_to_delete = ['Magnitude', 'meaningless_1', 'meaningless_2', 'star_ID',
#                    'Npts', 'CSSD']
names_to_delete = ['meaningless_1', 'meaningless_2', 'star_ID',
                   'Npts', 'CSSD', 'clipped_sigma', 'lag1', 'L', 'Lclp', 'Jclp',
                   'MAD', 'Ltim']

X, y, df, features_names, delta = load_data([file_0, file_1], names, names_to_delete)
target = 'variable'
predictors = list(df)
predictors.remove(target)
dtrain = df

kfold = StratifiedKFold(n_splits=4, shuffle=True, random_state=123)

def xg_f1(y, t):
    t = t.get_label()
    # Binaryzing your output
    y_bin = [1. if y_cont > 0.5 else 0. for y_cont in y]
    return 'f1', 1-f1_score(t, y_bin)

def objective(space):
    clf = xgb.XGBClassifier(n_estimators=10000, learning_rate=space['lr'],
                            max_depth=space['max_depth'],
                            min_child_weight=space['min_child_weight'],
                            subsample=space['subsample'],
                            colsample_bytree=space['colsample_bytree'],
                            colsample_bylevel=space['colsample_bylevel'],
                            gamma=space['gamma'],
                            scale_pos_weight=space['scale_pos_weight'])
                            # scale_pos_weight=space['scale_pos_weight'])

    # Try using pipeline
    estimators = list()
    estimators.append(('imputer', Imputer(missing_values='NaN', strategy='median',
                                          axis=0, verbose=2)))
    # estimators.append(('scaler', StandardScaler()))
    estimators.append(('clf', clf))
    pipeline = Pipeline(estimators)

    best_n = ""
    CMs = list()
    for train_indx, test_indx in kfold.split(dtrain[predictors].index,
                                             dtrain['variable']):
        train = dtrain.iloc[train_indx]
        valid = dtrain.iloc[test_indx]

        # X_test
        valid_ = valid[predictors]
        # X_train
        train_ = train[predictors]
        for name, transform in pipeline.steps[:-1]:
            transform.fit(train_)
            # X_test
            valid_ = transform.transform(valid_)
            # X_train
            train_ = transform.transform(train_)

        eval_set = [(train_, train['variable']),
                    (valid_, valid['variable'])]

        pipeline.fit(train[predictors], train['variable'],
                     # clf__eval_set=eval_set, clf__eval_metric="auc",
                     clf__eval_set=eval_set, clf__eval_metric=xg_f1,
                     clf__early_stopping_rounds=50)

        pred = pipeline.predict(valid[predictors])
        CMs.append(confusion_matrix(y[test_indx], pred))
        best_n = best_n + " " + str(clf.best_ntree_limit)

    CM = np.sum(CMs, axis=0)

    FN = CM[1][0]
    TP = CM[1][1]
    FP = CM[0][1]
    print "TP = {}".format(TP)
    print "FP = {}".format(FP)
    print "FN = {}".format(FN)

    f1 = 2. * TP / (2. * TP + FP + FN)
    print "F1 : {}".format(f1)

    return{'loss': 1-f1, 'status': STATUS_OK ,
           'attachments': {'best_n': best_n}}

space ={
    'max_depth': hp.choice("x_max_depth", np.arange(5, 12, 1, dtype=int)),
    'min_child_weight': hp.quniform('x_min_child', 1, 20, 2),
    'subsample': hp.quniform('x_subsample', 0.5, 1, 0.05),
    'colsample_bytree': hp.quniform('x_csbtree', 0.5, 1, 0.05),
    'colsample_bylevel': hp.quniform('x_csblevel', 0.5, 1, 0.05),
    'gamma': hp.quniform('x_gamma', 0.0, 1, 0.05),
    'scale_pos_weight': hp.quniform('x_spweight', 1, 303, 50),
    'lr': hp.quniform('lr', 0.001, 0.5, 0.025)
    # 'lr': hp.loguniform('lr', -7, -1)
}

trials = Trials()
best = fmin(fn=objective,
            space=space,
            algo=tpe.suggest,
            max_evals=300,
            trials=trials)

import hyperopt
print hyperopt.space_eval(space, best)

best_pars = hyperopt.space_eval(space, best)
best_n = trials.attachments['ATTACH::{}::best_n'.format(trials.best_trial['tid'])]
best_n = max([int(n) for n in best_n.strip().split(' ')])

clf = xgb.XGBClassifier(n_estimators=int(1.25 * best_n),
                        learning_rate=best_pars['lr'],
                        max_depth=best_pars['max_depth'],
                        min_child_weight=best_pars['min_child_weight'],
                        subsample=best_pars['subsample'],
                        colsample_bytree=best_pars['colsample_bytree'],
                        colsample_bylevel=best_pars['colsample_bylevel'],
                        gamma=best_pars['gamma'],
                        scale_pos_weight=best_pars['scale_pos_weight'])

estimators = list()
estimators.append(('imputer', Imputer(missing_values='NaN', strategy='median',
                                      axis=0, verbose=2)))
estimators.append(('clf', clf))
pipeline = Pipeline(estimators)

# Fit classifier with best hyperparameters on whole data set
pipeline.fit(dtrain[predictors], dtrain['variable'])

# Load blind test data
file_tgt = 'LMC_SC19_PSF_Pgood98__vast_lightcurve_statistics.log'
file_tgt = os.path.join(data_dir, file_tgt)
X_tgt, feature_names, df, df_orig = load_data_tgt(file_tgt, names, names_to_delete,
                                                  delta)

y_probs = pipeline.predict_proba(df[predictors])[:, 1]
idx = y_probs > 0.5
idx_ = y_probs < 0.5
gb_no = list(df_orig['star_ID'][idx_])
print("Found {} variables".format(np.count_nonzero(idx)))

with open('gb_results.txt', 'w') as fo:
    for line in list(df_orig['star_ID'][idx]):
        fo.write(line + '\n')

# Check F1
with open('clean_list_of_new_variables.txt', 'r') as fo:
    news = fo.readlines()
news = [line.strip().split(' ')[1] for line in news]
news = set(news)

with open('gb_results.txt', 'r') as fo:
    gb = fo.readlines()
gb = [line.strip().split('_')[4].split('.')[0] for line in gb]
gb = set(gb)

print "Among new vars found {}".format(len(news.intersection(gb)))

with open('candidates_50perc_threshold.txt', 'r') as fo:
    c50 = fo.readlines()
c50 = [line.strip("\", ', \", \n, }, {") for line in c50]

with open('variables_not_in_catalogs.txt', 'r') as fo:
    not_in_cat = fo.readlines()
nic = [line.strip().split(' ')[1] for line in not_in_cat]

# Catalogue variables
cat_vars = set(c50).difference(set(nic))
# Non-catalogue variable
noncat_vars = set([line.strip().split(' ')[1] for line in not_in_cat if 'CST' not in line])

# All variables
all_vars = news.union(cat_vars).union(noncat_vars)
gb_no = set([line.strip().split('_')[4].split('.')[0] for line in gb_no])

found_bad = '181193' in gb
print "Found known variable : ", found_bad

FN = len(gb_no.intersection(all_vars))
TP = len(all_vars.intersection(gb))
TN = len(gb_no) - FN
FP = len(gb) - TP
recall = float(TP) / (TP + FN)
precision = float(TP) / (TP + FP)
F1 = 2 * precision * recall / (precision + recall)
print "precision: {}".format(precision)
print "recall: {}".format(recall)
print "F1: {}".format(F1)
print "TN={}, FP={}".format(TN, FP)
print "FN={}, TP={}".format(FN, TP)

<|endoftext|>


<commit_before>import itemlib

class Inventory(object):

    def __init__(self, slots={}):
        self.call = itemlib.Items()
        self.slots = slots

    def bag(self):

        self.slots = {
                         self.call.item_id(1): 1
                     }

        sort = sorted(self.slots.items())
        for k, v in sort:
            if k == 0:
                if v == 0:
                    print('empty')
                else:
                    print(v)
            if v == 0:
                print('empty')
            else:
                print((k, v))
        return ' '<commit_msg>changing the invnetory method<commit_after>import itemlib

class Inventory(object):

    def __init__(self, slots={}):
        self.lib = itemlib.Items()
        self.slots = slots

    def bag(self):
        self.slots = {
                          'wood': 0,
                          'stone': 0,
                          'dirt': 0,
                          'gravel': 0,
                          'water': 0,
                          'wooden axe': 0,
                          'wooden sword': 0,
                          'stone axe': 0,
                          'stone sword': 0
                     }
        for k, v in sorted(self.slots.items()):
            print((k, v))
        return ' '<|endoftext|>