<commit_before>import plugintypes
import json
import subprocess
import uuid
import shutil
import re

import os
from os import path

import pip

from urllib.parse import urlparse

from tempfile import TemporaryFile

CENTRAL_REPO_URL="https://github.com/datamachine/telegram-pybot-plugin-repo"
CENTRAL_REPO_DIR="telegram-pybot-plugin-repo"

PLUGINS_REPOS_DIR="plugins.repos"
PLUGINS_TRASH_DIR="plugins.trash"

class PackageManagerPlugin(plugintypes.TelegramPlugin):
    """
    telegram-pybot's plugin package manager
    """
    patterns = [
        "^!pkg? (search) (.*)$",
        "^!pkg? (install) (.*)$",
        "^!pkg? (update)$",
        "^!pkg? (uninstall) ([\w_.-]+)$",
        "^!pkg? (installed)$",
    ]

    usage = [
        "!pkg search <query>: Search the repo for plugins",
        "!pkg update: Update the package repo cache",
        "!pkg upgrade: Update all plugins (not implemented)",
        "!pkg install <package name>: Install a package",
        "!pkg uninstall <package name>: Uninstall a package",
        "!pkg installed: List installed packages"
    ]

    @property
    def git_bin(self):
        if not self.has_option("git_bin"):
            self.write_option("git_bin", "/usr/bin/git")
        return self.read_option("git_bin")

    def __refresh_central_repo_object(self):
        try:
            with open(path.join(PLUGINS_REPOS_DIR, CENTRAL_REPO_DIR, "repo.json"), "r") as f:
                self.central_repo = json.load(f)
        except:
            print("Error opening repo.json")

    def activate_plugin(self):
        if not path.exists(PLUGINS_REPOS_DIR):
            os.makedirs(PLUGINS_REPOS_DIR)
        if PLUGINS_REPOS_DIR not in self.plugin_manager.getPluginLocator().plugins_places:
            self.plugin_manager.updatePluginPlaces([PLUGINS_REPOS_DIR])
            self.reload_plugins()
        else:
            self.__refresh_central_repo_object()

    def run(self, msg, matches):
        command = matches.group(1)
        if command == "install":
            return self.install_plugin(matches)

        if command == "uninstall":
            return self.uninstall_plugin(matches)

        if command == "search":
            return self.search_plugins(matches.group(2))

        if command == "update":
            return self.update_central_repo()

        if command == "installed":
            return self.get_installed()

    def __clone_repository(self, url):
            fp = TemporaryFile(mode="r")
            args = [self.git_bin, "clone", url]
            p = subprocess.Popen(args, cwd=PLUGINS_REPOS_DIR, stdout=fp, stderr=fp)
            code = p.wait()
            fp.seek(0)
            return (code, fp.read())

    def __get_repo_pkg_data(self, plugin_name):
        for pkg in self.central_repo["packages"]:
            if pkg["name"] == plugin_name:
                return pkg
        return None

    def __get_pkg_repo_path(self, pkg_name):
        for repo in os.listdir(PLUGINS_REPOS_DIR):
            repo_path = path.join(PLUGINS_REPOS_DIR, repo)
            repo_json_path = path.join(repo_path, "repository", "repo.json")
            try:
                with open(repo_json_path, 'r') as f:
                    repo_json = json.loads(f.read())
                    if repo_json["name"] == pkg_name:
                        print(repo_path)
                        return repo_path
                    continue
            except:
                continue
        return None

    def __get_pkg_requirements_path(self, pkg_name):
        pkg_repo_path = self.__get_pkg_repo_path(pkg_name)
        if not pkg_repo_path:
            return None

        return path.join(pkg_repo_path, "repository", "requirements.txt")

    def install_plugin(self, matches):
        plugin = matches.group(2)
        urldata = urlparse(matches.group(2))

        url = None
        if urldata.scheme in [ "http", "https" ]:
            url = plugin
        else:
            pkg = self.__get_repo_pkg_data(plugin)
            if pkg:
                url = pkg["repo"]

        if not url:
            return "Invalid plugin or url: {}".format(plugin)

        code, msg = self.__clone_repository(url)
        if code != 0:
            return msg

        pkg_req_path = self.__get_pkg_requirements_path(plugin)
        if pkg_req_path and os.path.exists(pkg_req_path):
            pip.main(['install', '-r', pkg_req_path])

        self.reload_plugins()

        return "{}\nSuccessfully installed plugin: {}".format(msg, plugin)

    def uninstall_plugin(self, matches):
        plugin_name = matches.group(2)
        for plugin in self.plugin_manager.getAllPlugins():
            if plugin.name != plugin_name:
                continue

            plugin_dir = path.relpath(path.dirname(plugin.path))
            if not plugin_dir.startswith(PLUGINS_REPOS_DIR):
                return "Error uninstalling plugin: {}.\nCannot locate plugin directory.".format(plugin_name)

            while plugin_dir and path.dirname(plugin_dir) != PLUGINS_REPOS_DIR:
                plugin_dir = path.dirname(plugin_dir)

            if not plugin_dir:
                return "Error uninstalling plugin: {}.\nCannot locate plugin directory.".format(plugin_name)

            self.plugin_manager.deactivatePluginByName(plugin_name)

            old_base = path.basename(plugin_dir)
            new_base = "{}.{}".format(path.basename(plugin_dir), str(uuid.uuid4()))
            new_dir = path.join(PLUGINS_TRASH_DIR, new_base)
            shutil.move(plugin_dir, new_dir)

            return "Uninstalled plugin: {}".format(plugin_name)
        return "Unable to find plugin: {}".format(plugin_name)

    def search_plugins(self, query):
        prog = re.compile(query, flags=re.IGNORECASE)
        results = ""
        for pkg in self.central_repo["packages"]:
            if prog.search(pkg["name"]) or prog.search(pkg["description"]):
                results += "{} | {} | {}\n".format(pkg["name"], pkg["version"], pkg["description"])
        return results

    def update_central_repo(self):
        central_repo_path = path.join(PLUGINS_REPOS_DIR, CENTRAL_REPO_DIR)
        f = TemporaryFile()
        if not path.exists(central_repo_path):
            args = [self.git_bin, "clone", CENTRAL_REPO_URL]
            p = subprocess.Popen(args, cwd=PLUGINS_REPOS_DIR, stdout=f, stderr=f)
            p.wait()
        else:
            args = [self.git_bin, "reset", "--hard"]
            p = subprocess.Popen(args, cwd=central_repo_path, stdout=f, stderr=f)
            p.wait()
            args = [self.git_bin, "pull"]
            p = subprocess.Popen(args, cwd=central_repo_path, stdout=f, stderr=f)
            p.wait()
        self.__refresh_central_repo_object()
        f.seek(0)
        return f.read().decode('utf-8')

    def get_installed(self):
        pkgs = ""
        for f in os.listdir(PLUGINS_REPOS_DIR):
            pkgs += "{}\n".format(f)
        return pkgs

    def reload_plugins(self):
        self.plugin_manager.collectPlugins()
        return "Plugins reloaded"

    def list_plugins(self):
        return os.listdir(PLUGINS_REPOS_DIR)

<commit_msg>more useful info in pkg installed<commit_after>import plugintypes
import json
import subprocess
import uuid
import shutil
import re

import os
from os import path

import pip

from urllib.parse import urlparse

from tempfile import TemporaryFile

CENTRAL_REPO_URL="https://github.com/datamachine/telegram-pybot-plugin-repo"
CENTRAL_REPO_DIR="telegram-pybot-plugin-repo"

PLUGINS_REPOS_DIR="plugins.repos"
PLUGINS_TRASH_DIR="plugins.trash"

class PackageManagerPlugin(plugintypes.TelegramPlugin):
    """
    telegram-pybot's plugin package manager
    """
    patterns = [
        "^!pkg? (search) (.*)$",
        "^!pkg? (install) (.*)$",
        "^!pkg? (update)$",
        "^!pkg? (uninstall) ([\w_.-]+)$",
        "^!pkg? (installed)$",
    ]

    usage = [
        "!pkg search <query>: Search the repo for plugins",
        "!pkg update: Update the package repo cache",
        "!pkg upgrade: Update all plugins (not implemented)",
        "!pkg install <package name>: Install a package",
        "!pkg uninstall <package name>: Uninstall a package",
        "!pkg installed: List installed packages"
    ]

    @property
    def git_bin(self):
        if not self.has_option("git_bin"):
            self.write_option("git_bin", "/usr/bin/git")
        return self.read_option("git_bin")

    def __refresh_central_repo_object(self):
        try:
            with open(path.join(PLUGINS_REPOS_DIR, CENTRAL_REPO_DIR, "repo.json"), "r") as f:
                self.central_repo = json.load(f)
        except:
            print("Error opening repo.json")

    def activate_plugin(self):
        if not path.exists(PLUGINS_REPOS_DIR):
            os.makedirs(PLUGINS_REPOS_DIR)
        if PLUGINS_REPOS_DIR not in self.plugin_manager.getPluginLocator().plugins_places:
            self.plugin_manager.updatePluginPlaces([PLUGINS_REPOS_DIR])
            self.reload_plugins()
        else:
            self.__refresh_central_repo_object()

    def run(self, msg, matches):
        command = matches.group(1)
        if command == "install":
            return self.install_plugin(matches)

        if command == "uninstall":
            return self.uninstall_plugin(matches)

        if command == "search":
            return self.search_plugins(matches.group(2))

        if command == "update":
            return self.update_central_repo()

        if command == "installed":
            return self.get_installed()

    def __clone_repository(self, url):
            fp = TemporaryFile(mode="r")
            args = [self.git_bin, "clone", url]
            p = subprocess.Popen(args, cwd=PLUGINS_REPOS_DIR, stdout=fp, stderr=fp)
            code = p.wait()
            fp.seek(0)
            return (code, fp.read())

    def __get_repo_pkg_data(self, plugin_name):
        for pkg in self.central_repo["packages"]:
            if pkg["name"] == plugin_name:
                return pkg
        return None

    def __get_pkg_repo_path(self, pkg_name):
        for repo in os.listdir(PLUGINS_REPOS_DIR):
            repo_path = path.join(PLUGINS_REPOS_DIR, repo)
            repo_json_path = path.join(repo_path, "repository", "repo.json")
            try:
                with open(repo_json_path, 'r') as f:
                    repo_json = json.loads(f.read())
                    if repo_json["name"] == pkg_name:
                        print(repo_path)
                        return repo_path
                    continue
            except:
                continue
        return None

    def __get_pkg_requirements_path(self, pkg_name):
        pkg_repo_path = self.__get_pkg_repo_path(pkg_name)
        if not pkg_repo_path:
            return None

        return path.join(pkg_repo_path, "repository", "requirements.txt")

    def __get_repo_json_from_repo_path(self, repo_path):
        repo_json_path = repo_json_path = path.join(repo_path, "repository", "repo.json")
        try:
            with open(repo_json_path, 'r') as f:
                return json.loads(f.read())
        except:
            pass
        return None

    def install_plugin(self, matches):
        plugin = matches.group(2)
        urldata = urlparse(matches.group(2))

        url = None
        if urldata.scheme in [ "http", "https" ]:
            url = plugin
        else:
            pkg = self.__get_repo_pkg_data(plugin)
            if pkg:
                url = pkg["repo"]

        if not url:
            return "Invalid plugin or url: {}".format(plugin)

        code, msg = self.__clone_repository(url)
        if code != 0:
            return msg

        pkg_req_path = self.__get_pkg_requirements_path(plugin)
        if pkg_req_path and os.path.exists(pkg_req_path):
            pip.main(['install', '-r', pkg_req_path])

        self.reload_plugins()

        return "{}\nSuccessfully installed plugin: {}".format(msg, plugin)

    def uninstall_plugin(self, matches):
        plugin_name = matches.group(2)
        for plugin in self.plugin_manager.getAllPlugins():
            if plugin.name != plugin_name:
                continue

            plugin_dir = path.relpath(path.dirname(plugin.path))
            if not plugin_dir.startswith(PLUGINS_REPOS_DIR):
                return "Error uninstalling plugin: {}.\nCannot locate plugin directory.".format(plugin_name)

            while plugin_dir and path.dirname(plugin_dir) != PLUGINS_REPOS_DIR:
                plugin_dir = path.dirname(plugin_dir)

            if not plugin_dir:
                return "Error uninstalling plugin: {}.\nCannot locate plugin directory.".format(plugin_name)

            self.plugin_manager.deactivatePluginByName(plugin_name)

            old_base = path.basename(plugin_dir)
            new_base = "{}.{}".format(path.basename(plugin_dir), str(uuid.uuid4()))
            new_dir = path.join(PLUGINS_TRASH_DIR, new_base)
            shutil.move(plugin_dir, new_dir)

            return "Uninstalled plugin: {}".format(plugin_name)
        return "Unable to find plugin: {}".format(plugin_name)

    def search_plugins(self, query):
        prog = re.compile(query, flags=re.IGNORECASE)
        results = ""
        for pkg in self.central_repo["packages"]:
            if prog.search(pkg["name"]) or prog.search(pkg["description"]):
                results += "{} | {} | {}\n".format(pkg["name"], pkg["version"], pkg["description"])
        return results

    def update_central_repo(self):
        central_repo_path = path.join(PLUGINS_REPOS_DIR, CENTRAL_REPO_DIR)
        f = TemporaryFile()
        if not path.exists(central_repo_path):
            args = [self.git_bin, "clone", CENTRAL_REPO_URL]
            p = subprocess.Popen(args, cwd=PLUGINS_REPOS_DIR, stdout=f, stderr=f)
            p.wait()
        else:
            args = [self.git_bin, "reset", "--hard"]
            p = subprocess.Popen(args, cwd=central_repo_path, stdout=f, stderr=f)
            p.wait()
            args = [self.git_bin, "pull"]
            p = subprocess.Popen(args, cwd=central_repo_path, stdout=f, stderr=f)
            p.wait()
        self.__refresh_central_repo_object()
        f.seek(0)
        return f.read().decode('utf-8')

    def get_installed(self):
        pkgs = ""
        for f in os.listdir(PLUGINS_REPOS_DIR):
            repo_path = os.path.join(PLUGINS_REPOS_DIR, f)
            repo_json = self.__get_repo_json_from_repo_path(repo_path)
            if repo_json:
                pkgs += "{} | {} | {}\n".format(repo_json["name"], repo_json["version"], repo_json["description"])
        return pkgs

    def reload_plugins(self):
        self.plugin_manager.collectPlugins()
        return "Plugins reloaded"

    def list_plugins(self):
        return os.listdir(PLUGINS_REPOS_DIR)

<|endoftext|>


<commit_before># Copyright (c) 2014, German Neuroinformatics Node (G-Node)
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

from __future__ import (absolute_import, division, print_function)#, unicode_literals)

import unittest
import sys
import numpy as np

from nix import *

class TestDataArray(unittest.TestCase):

    def setUp(self):
        self.file  = File.open("unittest.h5", FileMode.Overwrite)
        self.block = self.file.create_block("test block", "recordingsession")
        self.array = self.block.create_data_array("test array", "signal", DataType.Double, (100, ))
        self.other = self.block.create_data_array("other array", "signal", DataType.Double, (100, ))

    def tearDown(self):
        del self.file.blocks[self.block.id]
        self.file.close()

    def test_data_array_eq(self):
        assert(self.array == self.array)
        assert(not self.array == self.other)
        assert(not self.array == None)

    def test_data_array_id(self):
        assert(self.array.id is not None)

    def test_data_array_name(self):
        assert(self.array.name is not None)

    def test_data_array_type(self):
        def set_none():
            self.array.type = None

        assert(self.array.type is not None)
        self.assertRaises(Exception, set_none)

        self.array.type = "foo type"
        assert(self.array.type == "foo type")

    def test_data_array_definition(self):
        assert(self.array.definition is None)

        self.array.definition = "definition"
        assert(self.array.definition == "definition")

        self.array.definition = None
        assert(self.array.definition is None)

    def test_data_array_timestamps(self):
        created_at = self.array.created_at
        assert(created_at > 0)

        updated_at = self.array.updated_at
        assert(updated_at > 0)

        self.array.force_created_at(1403530068)
        assert(self.array.created_at == 1403530068)

    def test_data_array_label(self):
        assert(self.array.label is None)

        self.array.label = "label"
        assert(self.array.label == "label")

        self.array.label = None
        assert(self.array.label is None)

    def test_data_array_unit(self):
        assert(self.array.unit is None)

        self.array.unit = "mV"
        assert(self.array.unit == "mV")

        self.array.unit = None
        assert(self.array.unit is None)

    def test_data_array_exp_origin(self):
        assert(self.array.expansion_origin is None)

        self.array.expansion_origin = 10.2
        assert(self.array.expansion_origin == 10.2)

        self.array.expansion_origin = None
        assert(self.array.expansion_origin is None)

    def test_data_array_coefficients(self):
        assert(self.array.polynom_coefficients == ())

        self.array.polynom_coefficients = (1.1, 2.2)
        assert(self.array.polynom_coefficients == (1.1, 2.2))

        # TODO delete does not work

    def test_data_array_data(self):

        assert(self.array.polynom_coefficients == ())

        data = np.array([float(i) for i in range(100)])
        dout = np.empty_like(data)
        self.array.write_direct(data)
        assert(self.array.dtype == np.dtype(float))
        self.array.read_direct(dout)
        assert(np.array_equal(data, dout))
        dout = np.array(self.array)
        assert(np.array_equal(data, dout))
        assert(self.array.data_extent == data.shape)
        assert(self.array.data_extent == self.array.shape)
        assert(self.array.size == data.size)

        assert(len(self.array) == len(data))

        #indexing support in 1-d arrays
        self.assertRaises(IndexError, lambda : self.array[1:4:5])
        self.assertRaises(IndexError, lambda : self.array[[1,3,]])

        dout = np.array([self.array[i] for i in range(100)])
        assert(np.array_equal(data, dout))

        dout = self.array[...]
        assert(np.array_equal(data, dout))

        #indexed writing (1-d)
        data = np.array([float(-i) for i in range(100)])
        self.array[()] = data
        assert(np.array_equal(self.array[...], data))

        self.array[...] = [float(-i) for i in range(100)]
        assert(np.array_equal(self.array[()], data))
        assert(np.array_equal(self.array[0:-10], data[0:-10]))
        assert(np.array_equal(self.array[-10], data[-10]))

        self.array[0] = 42
        assert(self.array[0] == 42.0)

        #changing shape via data_extent property
        self.array.data_extent = (200, )
        assert(self.array.data_extent == (200, ))

        # TODO delete does not work
        data = np.eye(123)
        a1 = self.block.create_data_array("double array", "signal", DataType.Double, (123, 123))
        dset = a1
        dset.write_direct(data)
        dout = np.empty_like(data)
        dset.read_direct(dout)
        assert(np.array_equal(data, dout))

        #indexing support in 2-d arrays
        self.assertRaises(IndexError, lambda : self.array[[], [1,2]])

        dout = dset[12]
        assert(dout.shape == data[12].shape)
        assert(np.array_equal(dout, data[12]))
        assert(np.array_equal(dset[()], data))
        assert(np.array_equal(dset[...], data))
        assert(np.array_equal(dset[12, ...], data[12, ...]))
        assert(np.array_equal(dset[..., 12], data[..., 12]))
        assert(np.array_equal(dset[1:], data[1:]))
        assert(np.array_equal(dset[-20:, -20:], data[123-20:, 123-20:]))
        assert(np.array_equal(dset[:1], data[:1]))
        assert(np.array_equal(dset[:-1, :-1], data[1:123, 1:123]))
        assert(np.array_equal(dset[1:10, 1:10], data[1:10, 1:10]))
        assert(np.array_equal(dset[1:-2, 1:-2], data[1:121, 1:121]))

        a3 = self.block.create_data_array("int identity array", "signal", DataType.Int32, (123, 123))
        assert(a3.shape == (123, 123))
        assert(a3.dtype == np.dtype('i4'))

        data = np.random.rand(3, 4, 5)
        a4 = self.block.create_data_array("3d array", "signal", DataType.Double, (3, 4, 5))
        dset = a4
        dset.write_direct(data)
        assert(dset.shape == data.shape)
        assert(len(dset) == len(data))
        assert(dset.size == data.size)
        assert(np.array_equal(dset[2, ...], data[2, ...]))
        assert(np.array_equal(dset[-1, ...], data[2, ...]))
        assert(np.array_equal(dset[..., 3], data[..., 3]))
        assert(np.array_equal(dset[..., -2], data[..., 3]))
        assert(np.array_equal(dset[2, ..., 3], data[2, ..., 3]))
        assert(np.array_equal(dset[2, ..., -2], data[2, ..., 3]))
        assert(np.array_equal(dset[1:2, ..., 3:5], data[1:2, ..., 3:5]))
        assert(np.array_equal(dset[1:2, ..., 3:-1], data[1:2, ..., 3:4]))

        #indexed writing (n-d)
        d2 = np.random.rand(2,2)
        dset[1, 0:2, 0:2] = d2
        assert(np.array_equal(dset[1, 0:2, 0:2], d2))

        # test for the size check in DataSet.__len__
        # by simulating a system with a really smal int
        savemaxsize = sys.maxsize
        sys.maxsize = len(dset) - 1
        self.assertRaises(OverflowError, lambda : len(dset))
        sys.maxsize = savemaxsize

        # test inferring shape & dtype from data, and writing the data
        test_data = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0], dtype=int)
        da = self.block.create_data_array('created_from_data', 'b', data=test_data)
        assert(da.shape == test_data.shape)
        assert(np.array_equal(test_data, da[:]))

        #test for exceptions
        self.assertRaises(ValueError, lambda: self.block.create_data_array('x', 'y'))
        self.assertRaises(ValueError, lambda: self.block.create_data_array('x', 'y', data=test_data, shape=(1, 1, 1)))

        #test appending
        data = np.zeros((10, 5))
        da = self.block.create_data_array('append', 'double', data=data)
        to_append = np.zeros((2, 5))

        da.append(to_append)
        assert(da.shape == (12, 5))

        to_append = np.zeros((12, 2))
        da.append(to_append, axis=1)
        assert(da.shape == (12, 7))

        self.assertRaises(ValueError, lambda: da.append(np.zeros((3, 3, 3))))
        self.assertRaises(ValueError, lambda: da.append(np.zeros((5, 5))))

    def test_data_array_dtype(self):
        da = self.block.create_data_array('dtype_f8', 'b', 'f8', (10, 10))
        assert(da.dtype == np.dtype('f8'))

        da = self.block.create_data_array('dtype_i16', 'b', np.int16, (10, 10))
        data = da[:]
        assert(da.dtype == np.int16)
        assert(data.dtype == np.int16)

        da = self.block.create_data_array('dtype_int', 'b', int, (10, 10))
        assert(da.dtype == np.dtype(int))

        da = self.block.create_data_array('dtype_ndouble', 'b', DataType.Double, (10, 10))
        assert(da.dtype == np.dtype('f8'))

        da = self.block.create_data_array('dtype_auto', 'b', None, (10, 10))
        assert(da.dtype == np.dtype('f8'))

        test_data = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0], dtype=int)
        da = self.block.create_data_array('dtype_int_from_data', 'b', data=test_data)
        assert(da.dtype == test_data.dtype)

        void_data = np.array(['1', '2', '3', '4', '5', '6', '7', '8', '9', '0'], dtype='V1')
        da = self.block.create_data_array('dtype_opaque', 'b', data=void_data)
        assert(da.dtype == np.dtype('V1'))
        assert(np.array_equal(void_data, da[:]))

    def test_data_array_dimensions(self):
        assert(len(self.array.dimensions) == 0)

        setd    = self.array.append_set_dimension()
        ranged  = self.array.append_range_dimension(range(10))
        sampled = self.array.append_sampled_dimension(0.1)

        assert(len(self.array.dimensions) == 3)

        self.assertRaises(TypeError, lambda : self.array.dimensions["notexist"])
        self.assertRaises(KeyError, lambda : self.array.dimensions[-4])
        self.assertRaises(KeyError, lambda : self.array.dimensions[3])

        assert(isinstance(str(self.array.dimensions), basestring))
        assert(isinstance(repr(self.array.dimensions), basestring))

        dims   = list(self.array.dimensions)
        for i in range(3):
            assert(dims[i].index == self.array.dimensions[i].index)
            assert(dims[i].dimension_type == self.array.dimensions[i].dimension_type)

            assert(self.array.dimensions[i].index == self.array.dimensions[i - 3].index)

        del self.array.dimensions[2]
        del self.array.dimensions[1]
        del self.array.dimensions[0]

        assert(len(self.array.dimensions) == 0)

    def test_data_array_sources(self):
        source1 = self.block.create_source("source1", "channel")
        source2 = self.block.create_source("source2", "electrode")

        assert(len(self.array.sources) == 0)

        self.array.sources.append(source1)
        self.array.sources.append(source2)

        self.assertRaises(TypeError, lambda : self.array.sources.append(100))

        assert(len(self.array.sources) == 2)
        assert(source1 in self.array.sources)
        assert(source2 in self.array.sources)

        del self.array.sources[source2]
        assert(self.array.sources[0] == source1)

        del self.array.sources[source1]
        assert(len(self.array.sources) == 0)
<commit_msg>[TestDataArray]  add test for aliasRange<commit_after># Copyright (c) 2014, German Neuroinformatics Node (G-Node)
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

from __future__ import (absolute_import, division, print_function)#, unicode_literals)

import unittest
import sys
import numpy as np

from nix import *

class TestDataArray(unittest.TestCase):

    def setUp(self):
        self.file  = File.open("unittest.h5", FileMode.Overwrite)
        self.block = self.file.create_block("test block", "recordingsession")
        self.array = self.block.create_data_array("test array", "signal", DataType.Double, (100, ))
        self.other = self.block.create_data_array("other array", "signal", DataType.Double, (100, ))

    def tearDown(self):
        del self.file.blocks[self.block.id]
        self.file.close()

    def test_data_array_eq(self):
        assert(self.array == self.array)
        assert(not self.array == self.other)
        assert(not self.array == None)

    def test_data_array_id(self):
        assert(self.array.id is not None)

    def test_data_array_name(self):
        assert(self.array.name is not None)

    def test_data_array_type(self):
        def set_none():
            self.array.type = None

        assert(self.array.type is not None)
        self.assertRaises(Exception, set_none)

        self.array.type = "foo type"
        assert(self.array.type == "foo type")

    def test_data_array_definition(self):
        assert(self.array.definition is None)

        self.array.definition = "definition"
        assert(self.array.definition == "definition")

        self.array.definition = None
        assert(self.array.definition is None)

    def test_data_array_timestamps(self):
        created_at = self.array.created_at
        assert(created_at > 0)

        updated_at = self.array.updated_at
        assert(updated_at > 0)

        self.array.force_created_at(1403530068)
        assert(self.array.created_at == 1403530068)

    def test_data_array_label(self):
        assert(self.array.label is None)

        self.array.label = "label"
        assert(self.array.label == "label")

        self.array.label = None
        assert(self.array.label is None)

    def test_data_array_unit(self):
        assert(self.array.unit is None)

        self.array.unit = "mV"
        assert(self.array.unit == "mV")

        self.array.unit = None
        assert(self.array.unit is None)

    def test_data_array_exp_origin(self):
        assert(self.array.expansion_origin is None)

        self.array.expansion_origin = 10.2
        assert(self.array.expansion_origin == 10.2)

        self.array.expansion_origin = None
        assert(self.array.expansion_origin is None)

    def test_data_array_coefficients(self):
        assert(self.array.polynom_coefficients == ())

        self.array.polynom_coefficients = (1.1, 2.2)
        assert(self.array.polynom_coefficients == (1.1, 2.2))

        # TODO delete does not work

    def test_data_array_data(self):

        assert(self.array.polynom_coefficients == ())

        data = np.array([float(i) for i in range(100)])
        dout = np.empty_like(data)
        self.array.write_direct(data)
        assert(self.array.dtype == np.dtype(float))
        self.array.read_direct(dout)
        assert(np.array_equal(data, dout))
        dout = np.array(self.array)
        assert(np.array_equal(data, dout))
        assert(self.array.data_extent == data.shape)
        assert(self.array.data_extent == self.array.shape)
        assert(self.array.size == data.size)

        assert(len(self.array) == len(data))

        #indexing support in 1-d arrays
        self.assertRaises(IndexError, lambda : self.array[1:4:5])
        self.assertRaises(IndexError, lambda : self.array[[1,3,]])

        dout = np.array([self.array[i] for i in range(100)])
        assert(np.array_equal(data, dout))

        dout = self.array[...]
        assert(np.array_equal(data, dout))

        #indexed writing (1-d)
        data = np.array([float(-i) for i in range(100)])
        self.array[()] = data
        assert(np.array_equal(self.array[...], data))

        self.array[...] = [float(-i) for i in range(100)]
        assert(np.array_equal(self.array[()], data))
        assert(np.array_equal(self.array[0:-10], data[0:-10]))
        assert(np.array_equal(self.array[-10], data[-10]))

        self.array[0] = 42
        assert(self.array[0] == 42.0)

        #changing shape via data_extent property
        self.array.data_extent = (200, )
        assert(self.array.data_extent == (200, ))

        # TODO delete does not work
        data = np.eye(123)
        a1 = self.block.create_data_array("double array", "signal", DataType.Double, (123, 123))
        dset = a1
        dset.write_direct(data)
        dout = np.empty_like(data)
        dset.read_direct(dout)
        assert(np.array_equal(data, dout))

        #indexing support in 2-d arrays
        self.assertRaises(IndexError, lambda : self.array[[], [1,2]])

        dout = dset[12]
        assert(dout.shape == data[12].shape)
        assert(np.array_equal(dout, data[12]))
        assert(np.array_equal(dset[()], data))
        assert(np.array_equal(dset[...], data))
        assert(np.array_equal(dset[12, ...], data[12, ...]))
        assert(np.array_equal(dset[..., 12], data[..., 12]))
        assert(np.array_equal(dset[1:], data[1:]))
        assert(np.array_equal(dset[-20:, -20:], data[123-20:, 123-20:]))
        assert(np.array_equal(dset[:1], data[:1]))
        assert(np.array_equal(dset[:-1, :-1], data[1:123, 1:123]))
        assert(np.array_equal(dset[1:10, 1:10], data[1:10, 1:10]))
        assert(np.array_equal(dset[1:-2, 1:-2], data[1:121, 1:121]))

        a3 = self.block.create_data_array("int identity array", "signal", DataType.Int32, (123, 123))
        assert(a3.shape == (123, 123))
        assert(a3.dtype == np.dtype('i4'))

        data = np.random.rand(3, 4, 5)
        a4 = self.block.create_data_array("3d array", "signal", DataType.Double, (3, 4, 5))
        dset = a4
        dset.write_direct(data)
        assert(dset.shape == data.shape)
        assert(len(dset) == len(data))
        assert(dset.size == data.size)
        assert(np.array_equal(dset[2, ...], data[2, ...]))
        assert(np.array_equal(dset[-1, ...], data[2, ...]))
        assert(np.array_equal(dset[..., 3], data[..., 3]))
        assert(np.array_equal(dset[..., -2], data[..., 3]))
        assert(np.array_equal(dset[2, ..., 3], data[2, ..., 3]))
        assert(np.array_equal(dset[2, ..., -2], data[2, ..., 3]))
        assert(np.array_equal(dset[1:2, ..., 3:5], data[1:2, ..., 3:5]))
        assert(np.array_equal(dset[1:2, ..., 3:-1], data[1:2, ..., 3:4]))

        #indexed writing (n-d)
        d2 = np.random.rand(2,2)
        dset[1, 0:2, 0:2] = d2
        assert(np.array_equal(dset[1, 0:2, 0:2], d2))

        # test for the size check in DataSet.__len__
        # by simulating a system with a really smal int
        savemaxsize = sys.maxsize
        sys.maxsize = len(dset) - 1
        self.assertRaises(OverflowError, lambda : len(dset))
        sys.maxsize = savemaxsize

        # test inferring shape & dtype from data, and writing the data
        test_data = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0], dtype=int)
        da = self.block.create_data_array('created_from_data', 'b', data=test_data)
        assert(da.shape == test_data.shape)
        assert(np.array_equal(test_data, da[:]))

        #test for exceptions
        self.assertRaises(ValueError, lambda: self.block.create_data_array('x', 'y'))
        self.assertRaises(ValueError, lambda: self.block.create_data_array('x', 'y', data=test_data, shape=(1, 1, 1)))

        #test appending
        data = np.zeros((10, 5))
        da = self.block.create_data_array('append', 'double', data=data)
        to_append = np.zeros((2, 5))

        da.append(to_append)
        assert(da.shape == (12, 5))

        to_append = np.zeros((12, 2))
        da.append(to_append, axis=1)
        assert(da.shape == (12, 7))

        self.assertRaises(ValueError, lambda: da.append(np.zeros((3, 3, 3))))
        self.assertRaises(ValueError, lambda: da.append(np.zeros((5, 5))))

    def test_data_array_dtype(self):
        da = self.block.create_data_array('dtype_f8', 'b', 'f8', (10, 10))
        assert(da.dtype == np.dtype('f8'))

        da = self.block.create_data_array('dtype_i16', 'b', np.int16, (10, 10))
        data = da[:]
        assert(da.dtype == np.int16)
        assert(data.dtype == np.int16)

        da = self.block.create_data_array('dtype_int', 'b', int, (10, 10))
        assert(da.dtype == np.dtype(int))

        da = self.block.create_data_array('dtype_ndouble', 'b', DataType.Double, (10, 10))
        assert(da.dtype == np.dtype('f8'))

        da = self.block.create_data_array('dtype_auto', 'b', None, (10, 10))
        assert(da.dtype == np.dtype('f8'))

        test_data = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0], dtype=int)
        da = self.block.create_data_array('dtype_int_from_data', 'b', data=test_data)
        assert(da.dtype == test_data.dtype)

        void_data = np.array(['1', '2', '3', '4', '5', '6', '7', '8', '9', '0'], dtype='V1')
        da = self.block.create_data_array('dtype_opaque', 'b', data=void_data)
        assert(da.dtype == np.dtype('V1'))
        assert(np.array_equal(void_data, da[:]))

    def test_data_array_dimensions(self):
        assert(len(self.array.dimensions) == 0)

        setd    = self.array.append_set_dimension()
        ranged  = self.array.append_range_dimension(range(10))
        sampled = self.array.append_sampled_dimension(0.1)

        assert(len(self.array.dimensions) == 3)

        self.assertRaises(TypeError, lambda : self.array.dimensions["notexist"])
        self.assertRaises(KeyError, lambda : self.array.dimensions[-4])
        self.assertRaises(KeyError, lambda : self.array.dimensions[3])

        assert(isinstance(str(self.array.dimensions), basestring))
        assert(isinstance(repr(self.array.dimensions), basestring))

        dims   = list(self.array.dimensions)
        for i in range(3):
            assert(dims[i].index == self.array.dimensions[i].index)
            assert(dims[i].dimension_type == self.array.dimensions[i].dimension_type)

            assert(self.array.dimensions[i].index == self.array.dimensions[i - 3].index)

        del self.array.dimensions[2]
        del self.array.dimensions[1]
        del self.array.dimensions[0]

        assert(len(self.array.dimensions) == 0)
        self.array.append_alias_range_dimension()
        assert(len(self.array.dimensions) == 1)
        del self.array.dimensions[0]
        self.array.create_alias_range_dimension()
        assert(len(self.array.dimensions) == 1)

        self.assertRaises(ValueError, lambda : self.array.append_alias_range_dimension())
        self.assertRaises(ValueError, lambda : self.array.create_alias_range_dimension())
        string_array = self.block.create_data_array('string_array', 'nix.texts', dtype=DataType.String, shape=(10,))
        self.assertRaises(ValueError, lambda : string_array.append_alias_range_dimension())
        self.assertRaises(ValueError, lambda : string_array.create_alias_range_dimension())
        assert(len(string_array.dimensions) == 0)
        del self.block.data_arrays['string_array']

        array_2D = self.block.create_data_array('array_2d', 'nix.2d', dtype=DataType.Double, shape=(10,10))
        self.assertRaises(ValueError, lambda : array_2D.append_alias_range_dimension())
        self.assertRaises(ValueError, lambda : array_2D.create_alias_range_dimension())
        assert(len(array_2D.dimensions) == 0)
        del self.block.data_arrays['array_2d']

    def test_data_array_sources(self):
        source1 = self.block.create_source("source1", "channel")
        source2 = self.block.create_source("source2", "electrode")

        assert(len(self.array.sources) == 0)

        self.array.sources.append(source1)
        self.array.sources.append(source2)

        self.assertRaises(TypeError, lambda : self.array.sources.append(100))

        assert(len(self.array.sources) == 2)
        assert(source1 in self.array.sources)
        assert(source2 in self.array.sources)

        del self.array.sources[source2]
        assert(self.array.sources[0] == source1)

        del self.array.sources[source1]
        assert(len(self.array.sources) == 0)
<|endoftext|>


<commit_before>
from setuptools import setup, find_packages

version = '5.0.4'

setup(
    name="alerta-telegram",
    version=version,
    description='Alerta plugin for Telegram',
    url='https://github.com/alerta/alerta-contrib',
    license='MIT',
    author='Nick Satterly',
    author_email='nick.satterly@theguardian.com',
    packages=find_packages(),
    py_modules=['alerta_telegram'],
    install_requires=[
        'telepot',
        'jinja2'
    ],
    include_package_data=True,
    zip_safe=True,
    entry_points={
        'alerta.plugins': [
            'telegram = alerta_telegram:TelegramBot'
        ]
    }
)
<commit_msg>Bump version to 5.1.0<commit_after>
from setuptools import setup, find_packages

version = '5.1.0'

setup(
    name="alerta-telegram",
    version=version,
    description='Alerta plugin for Telegram',
    url='https://github.com/alerta/alerta-contrib',
    license='MIT',
    author='Nick Satterly',
    author_email='nick.satterly@theguardian.com',
    packages=find_packages(),
    py_modules=['alerta_telegram'],
    install_requires=[
        'telepot',
        'jinja2'
    ],
    include_package_data=True,
    zip_safe=True,
    entry_points={
        'alerta.plugins': [
            'telegram = alerta_telegram:TelegramBot'
        ]
    }
)
<|endoftext|>


<commit_before>from decimal import Decimal, InvalidOperation
from datetime import datetime, date, time

import warnings
warnings.filterwarnings('ignore', '^Table.+already exists$', Warning, '^pony\\.orm$')

import MySQLdb
from MySQLdb import (Warning, Error, InterfaceError, DatabaseError,
                     DataError, OperationalError, IntegrityError, InternalError,
                     ProgrammingError, NotSupportedError)
from MySQLdb.constants import FIELD_TYPE, FLAG
import MySQLdb.converters

from pony import dbschema
from pony import sqlbuilding
from pony.sqltranslation import SQLTranslator
from pony.clobtypes import LongStr, LongUnicode
from pony.utils import localbase

paramstyle = 'format'

class MySQLColumn(dbschema.Column):
    autoincrement = 'AUTO_INCREMENT'

class MySQLSchema(dbschema.DBSchema):
    column_class = MySQLColumn

def create_schema(database):
    return MySQLSchema(database)

def quote_name(connection, name):
    return sqlbuilding.quote_name(name, "`")

def get_pool(*args, **keyargs):
    if 'conv' not in keyargs:
        conv = MySQLdb.converters.conversions.copy()
        conv[FIELD_TYPE.BLOB] = [(FLAG.BINARY, buffer)]
        keyargs['conv'] = conv
    if 'charset' not in keyargs:
        keyargs['charset'] = 'utf8'
    return Pool(*args, **keyargs)

class Pool(localbase):
    def __init__(pool, *args, **keyargs): # called separately in each thread
        pool.args = args
        pool.keyargs = keyargs
        pool.con = None
    def connect(pool):
        if pool.con is None:
            pool.con = MySQLdb.connect(*pool.args, **pool.keyargs)
        return pool.con
    def release(pool, con):
        assert con is pool.con
        try: con.rollback()
        except:
            pool.close(con)
            raise
    def close(pool, con):
        assert con is pool.con
        pool.con = None
        con.close()

class MySQLTranslator(SQLTranslator):
    def DateMixin_attr_year(translator, monad):
        sql = [ 'YEAR', monad.getsql()[0] ]
        translator = monad.translator
        return translator.NumericExprMonad(translator, int, sql)

translator_cls = MySQLTranslator

class MyValue(sqlbuilding.Value):
    def quote_str(self, s):
        s = s.replace('%', '%%')
        return sqlbuilding.Value.quote_str(self, s)

class MySQLBuilder(sqlbuilding.SQLBuilder):
    make_value = MyValue
    def YEAR(builder, expr):
        return 'year(', builder(expr), ')'

def ast2sql(con, ast):
    b = MySQLBuilder(ast, paramstyle, "`")
    return b.sql, b.adapter

def get_last_rowid(cursor):
    return cursor.lastrowid

def _get_converter_type_by_py_type(py_type):
    if issubclass(py_type, bool): return BoolConverter
    elif issubclass(py_type, unicode): return UnicodeConverter
    elif issubclass(py_type, str): return StrConverter
    elif issubclass(py_type, (int, long)): return IntConverter
    elif issubclass(py_type, float): return RealConverter
    elif issubclass(py_type, Decimal): return DecimalConverter
    elif issubclass(py_type, buffer): return BlobConverter
    elif issubclass(py_type, datetime): return DatetimeConverter
    elif issubclass(py_type, date): return DateConverter
    else: raise TypeError, py_type

def get_converter_by_py_type(py_type):
    return _get_converter_type_by_py_type(py_type)()

def get_converter_by_attr(attr):
    return _get_converter_type_by_py_type(attr.py_type)(attr)

class Converter(object):
    def __init__(converter, attr=None):
        converter.attr = attr
        if attr is None: return
        keyargs = attr.keyargs.copy()
        converter.init(keyargs)
        for option in keyargs: raise TypeError('Unknown option %r' % option)
    def init(converter, keyargs):
        pass
    def validate(converter, val):
        return val
    def py2sql(converter, val):
        return val
    def sql2py(converter, val):
        return val

class BoolConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
    def validate(converter, val):
        return bool(val)
    def sql2py(converter, val):
        return bool(val)
    def sql_type(converter):
        return "BOOLEAN"

class BasestringConverter(Converter):
    def __init__(converter, attr=None):
        converter.db_encoding = None
        Converter.__init__(converter, attr)
    def init(converter, keyargs):
        attr = converter.attr
        if attr:
            if not attr.args: max_len = None
            elif len(attr.args) > 1: unexpected_args(attr, attr.args[1:])
            else: max_len = attr.args[0]
            if issubclass(attr.py_type, (LongStr, LongUnicode)):
                if max_len is not None: raise TypeError('Max length is not supported for CLOBs')
            elif max_len is None: max_len = 200
            elif not isinstance(max_len, (int, long)):
                raise TypeError('Max length argument must be int. Got: %r' % max_len)
            converter.max_len = max_len
        else: converter.max_len = None
        converter.db_encoding = keyargs.pop('db_encoding', 'utf8')
    def validate(converter, val):
        max_len = converter.max_len
        val_len = len(val)
        if max_len and val_len > max_len:
            raise ValueError('Value for attribute %s is too long. Max length is %d, value length is %d'
                             % (converter.attr, max_len, val_len))
        if not val_len: raise ValueError('Empty strings are not allowed. Try using None instead')
        return val
    def sql_type(converter):
        if converter.max_len:
            return 'VARCHAR(%d) CHARACTER SET %s' % (converter.max_len, converter.db_encoding)
        return 'TEXT CHARACTER SET %s' % converter.db_encoding

class UnicodeConverter(BasestringConverter):
    def validate(converter, val):
        if val is None: pass
        elif isinstance(val, str): val = val.decode('ascii')
        elif not isinstance(val, unicode): raise TypeError(
            'Value type for attribute %s must be unicode. Got: %r' % (converter.attr, type(val)))
        return BasestringConverter.validate(converter, val)

class StrConverter(BasestringConverter):
    def __init__(converter, attr=None):
        converter.encoding = 'ascii'  # for the case when attr is None
        BasestringConverter.__init__(converter, attr)
    def init(converter, keyargs):
        BasestringConverter.init(converter, keyargs)
        converter.encoding = keyargs.pop('encoding', 'latin1')
    def validate(converter, val):
        if val is not None:
            if isinstance(val, str): pass
            elif isinstance(val, unicode): val = val.encode(converter.encoding)
            else: raise TypeError('Value type for attribute %s must be str in encoding %r. Got: %r'
                                  % (converter.attr, converter.encoding, type(val)))
        return BasestringConverter.validate(converter, val)
    def py2sql(converter, val):
        return val.decode(converter.encoding)
    def sql2py(converter, val):
        return val.encode(converter.encoding, 'replace')

class IntConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
        min_val = keyargs.pop('min', None)
        if min_val is not None and not isinstance(min_val, (int, long)):
            raise TypeError("'min' argument for attribute %s must be int. Got: %r" % (attr, min_val))
        max_val = keyargs.pop('max', None)
        if max_val is not None and not isinstance(max_val, (int, long)):
            raise TypeError("'max' argument for attribute %s must be int. Got: %r" % (attr, max_val))
        converter.min_val = min_val
        converter.max_val = max_val
    def validate(converter, val):
        if not isinstance(val, (int, long)):
            raise TypeError('Value type for attribute %s must be int. Got: %r' % (converter.attr, type(val)))
        if converter.min_val and val < converter.min_val:
            raise ValueError('Value %r of attr %s is less than the minimum allowed value %r'
                             % (val, converter.attr, converter.min_val))
        if converter.max_val and val > converter.max_val:
            raise ValueError('Value %r of attr %s is greater than the maximum allowed value %r'
                             % (val, converter.attr, converter.max_val))
        return val
    def sql_type(converter):
        return 'INTEGER'

class RealConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
        min_val = keyargs.pop('min', None)
        if min_val is not None:
            try: min_val = float(min_val)
            except ValueError:
                raise TypeError("Invalid value for 'min' argument for attribute %s: %r" % (attr, min_val))
        max_val = keyargs.pop('max', None)
        if max_val is not None:
            try: max_val = float(max_val)
            except ValueError:
                raise TypeError("Invalid value for 'max' argument for attribute %s: %r" % (attr, max_val))
        converter.min_val = min_val
        converter.max_val = max_val
    def validate(converter, val):
        try: val = float(val)
        except ValueError:
            raise TypeError('Invalid value for attribute %s: %r' % (converter.attr, val))
        if converter.min_val and val < converter.min_val:
            raise ValueError('Value %r of attr %s is less than the minimum allowed value %r'
                             % (val, converter.attr, converter.min_val))
        if converter.max_val and val > converter.max_val:
            raise ValueError('Value %r of attr %s is greater than the maximum allowed value %r'
                             % (val, converter.attr, converter.max_val))
        return val
    def sql_type(converter):
        return 'DOUBLE'

class DecimalConverter(Converter):
    def __init__(converter, attr=None):
        Converter.__init__(converter, attr)
    def init(converter, keyargs):
        attr = converter.attr
        args = attr.args
        if len(args) > 2: raise TypeError('Too many positional parameters for Decimal (expected: precision and scale)')

        if args: precision = args[0]
        else: precision = keyargs.pop('precision', 12)
        if not isinstance(precision, (int, long)):
            raise TypeError("'precision' positional argument for attribute %s must be int. Got: %r" % (attr, precision))
        if precision <= 0: raise TypeError(
            "'precision' positional argument for attribute %s must be positive. Got: %r" % (attr, precision))

        if len(args) == 2: scale = args[1]
        else: scale = keyargs.pop('scale', 2)
        if not isinstance(scale, (int, long)):
            raise TypeError("'scale' positional argument for attribute %s must be int. Got: %r" % (attr, scale))
        if scale <= 0: raise TypeError(
            "'scale' positional argument for attribute %s must be positive. Got: %r" % (attr, scale))

        if scale > precision: raise ValueError("'scale' must be less or equal 'precision'")
        converter.precision = precision
        converter.scale = scale
        converter.exp = Decimal(10) ** -scale

        min_val = keyargs.pop('min', None)
        if min_val is not None:
            try: min_val = Decimal(min_val)
            except TypeError: raise TypeError(
                "Invalid value for 'min' argument for attribute %s: %r" % (attr, min_val))

        max_val = keyargs.pop('max', None)
        if max_val is not None:
            try: max_val = Decimal(max_val)
            except TypeError: raise TypeError(
                "Invalid value for 'max' argument for attribute %s: %r" % (attr, max_val))

        converter.min_val = min_val
        converter.max_val = max_val
    def validate(converter, val):
        if type(val) is Decimal: return val
        try: return Decimal(val)
        except InvalidOperation, exc:
            raise TypeError('Invalid value for attribute %s: %r' % (converter.attr, val))
        if converter.min_val is not None and val < converter.min_val:
            raise ValueError('Value %r of attr %s is less than the minimum allowed value %r'
                             % (val, converter.attr, converter.min_val))
        if converter.max_val is not None and val > converter.max_val:
            raise ValueError('Value %r of attr %s is greater than the maximum allowed value %r'
                             % (val, converter.attr, converter.max_val))
    def sql_type(converter):
        return 'DECIMAL(%d, %d)' % (converter.precision, converter.scale)

class BlobConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
    def validate(converter, val):
        if isinstance(val, buffer): return val
        if isinstance(val, str): return buffer(val)
        raise TypeError("Attribute %r: expected type is 'buffer'. Got: %r" % (converter.attr, type(val)))
    def sql_type(converter):
        return 'BLOB'

class DatetimeConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
    def validate(converter, val):
        if not isinstance(val, datetime):
            raise TypeError("Attribute %r: expected type is 'datetime'. Got: %r" % (converter.attr, val))
        return val
    def sql_type(converter):
        return 'DATETIME'

class DateConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
    def validate(converter, val):
        if isinstance(val, datetime): return val.date()
        if not isinstance(val, date):
            raise TypeError("Attribute %r: expected type is 'date'. Got: %r" % (converter.attr, val))
        return val
    def sql_type(converter):
        return 'DATE'<commit_msg>mysql provider: bigint and longblob types support<commit_after>from decimal import Decimal, InvalidOperation
from datetime import datetime, date, time

import warnings
warnings.filterwarnings('ignore', '^Table.+already exists$', Warning, '^pony\\.orm$')

import MySQLdb
from MySQLdb import (Warning, Error, InterfaceError, DatabaseError,
                     DataError, OperationalError, IntegrityError, InternalError,
                     ProgrammingError, NotSupportedError)
from MySQLdb.constants import FIELD_TYPE, FLAG
import MySQLdb.converters

from pony import dbschema
from pony import sqlbuilding
from pony.sqltranslation import SQLTranslator
from pony.clobtypes import LongStr, LongUnicode
from pony.utils import localbase

paramstyle = 'format'

class MySQLColumn(dbschema.Column):
    autoincrement = 'AUTO_INCREMENT'

class MySQLSchema(dbschema.DBSchema):
    column_class = MySQLColumn

def create_schema(database):
    return MySQLSchema(database)

def quote_name(connection, name):
    return sqlbuilding.quote_name(name, "`")

def get_pool(*args, **keyargs):
    if 'conv' not in keyargs:
        conv = MySQLdb.converters.conversions.copy()
        conv[FIELD_TYPE.BLOB] = [(FLAG.BINARY, buffer)]
        keyargs['conv'] = conv
    if 'charset' not in keyargs:
        keyargs['charset'] = 'utf8'
    return Pool(*args, **keyargs)

class Pool(localbase):
    def __init__(pool, *args, **keyargs): # called separately in each thread
        pool.args = args
        pool.keyargs = keyargs
        pool.con = None
    def connect(pool):
        if pool.con is None:
            pool.con = MySQLdb.connect(*pool.args, **pool.keyargs)
        return pool.con
    def release(pool, con):
        assert con is pool.con
        try: con.rollback()
        except:
            pool.close(con)
            raise
    def close(pool, con):
        assert con is pool.con
        pool.con = None
        con.close()

class MySQLTranslator(SQLTranslator):
    def DateMixin_attr_year(translator, monad):
        sql = [ 'YEAR', monad.getsql()[0] ]
        translator = monad.translator
        return translator.NumericExprMonad(translator, int, sql)

translator_cls = MySQLTranslator

class MyValue(sqlbuilding.Value):
    def quote_str(self, s):
        s = s.replace('%', '%%')
        return sqlbuilding.Value.quote_str(self, s)

class MySQLBuilder(sqlbuilding.SQLBuilder):
    make_value = MyValue
    def YEAR(builder, expr):
        return 'year(', builder(expr), ')'

def ast2sql(con, ast):
    b = MySQLBuilder(ast, paramstyle, "`")
    return b.sql, b.adapter

def get_last_rowid(cursor):
    return cursor.lastrowid

def _get_converter_type_by_py_type(py_type):
    if issubclass(py_type, bool): return BoolConverter
    elif issubclass(py_type, unicode): return UnicodeConverter
    elif issubclass(py_type, str): return StrConverter
    elif issubclass(py_type, int): return IntConverter
    elif issubclass(py_type, long): return LongConverter
    elif issubclass(py_type, float): return RealConverter
    elif issubclass(py_type, Decimal): return DecimalConverter
    elif issubclass(py_type, buffer): return BlobConverter
    elif issubclass(py_type, datetime): return DatetimeConverter
    elif issubclass(py_type, date): return DateConverter
    else: raise TypeError, py_type

def get_converter_by_py_type(py_type):
    return _get_converter_type_by_py_type(py_type)()

def get_converter_by_attr(attr):
    return _get_converter_type_by_py_type(attr.py_type)(attr)

class Converter(object):
    def __init__(converter, attr=None):
        converter.attr = attr
        if attr is None: return
        keyargs = attr.keyargs.copy()
        converter.init(keyargs)
        for option in keyargs: raise TypeError('Unknown option %r' % option)
    def init(converter, keyargs):
        pass
    def validate(converter, val):
        return val
    def py2sql(converter, val):
        return val
    def sql2py(converter, val):
        return val

class BoolConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
    def validate(converter, val):
        return bool(val)
    def sql2py(converter, val):
        return bool(val)
    def sql_type(converter):
        return "BOOLEAN"

class BasestringConverter(Converter):
    def __init__(converter, attr=None):
        converter.db_encoding = None
        Converter.__init__(converter, attr)
    def init(converter, keyargs):
        attr = converter.attr
        if attr:
            if not attr.args: max_len = None
            elif len(attr.args) > 1: unexpected_args(attr, attr.args[1:])
            else: max_len = attr.args[0]
            if issubclass(attr.py_type, (LongStr, LongUnicode)):
                if max_len is not None: raise TypeError('Max length is not supported for CLOBs')
            elif max_len is None: max_len = 200
            elif not isinstance(max_len, (int, long)):
                raise TypeError('Max length argument must be int. Got: %r' % max_len)
            converter.max_len = max_len
        else: converter.max_len = None
        converter.db_encoding = keyargs.pop('db_encoding', 'utf8')
    def validate(converter, val):
        max_len = converter.max_len
        val_len = len(val)
        if max_len and val_len > max_len:
            raise ValueError('Value for attribute %s is too long. Max length is %d, value length is %d'
                             % (converter.attr, max_len, val_len))
        if not val_len: raise ValueError('Empty strings are not allowed. Try using None instead')
        return val
    def sql_type(converter):
        if converter.max_len:
            return 'VARCHAR(%d) CHARACTER SET %s' % (converter.max_len, converter.db_encoding)
        return 'TEXT CHARACTER SET %s' % converter.db_encoding

class UnicodeConverter(BasestringConverter):
    def validate(converter, val):
        if val is None: pass
        elif isinstance(val, str): val = val.decode('ascii')
        elif not isinstance(val, unicode): raise TypeError(
            'Value type for attribute %s must be unicode. Got: %r' % (converter.attr, type(val)))
        return BasestringConverter.validate(converter, val)

class StrConverter(BasestringConverter):
    def __init__(converter, attr=None):
        converter.encoding = 'ascii'  # for the case when attr is None
        BasestringConverter.__init__(converter, attr)
    def init(converter, keyargs):
        BasestringConverter.init(converter, keyargs)
        converter.encoding = keyargs.pop('encoding', 'latin1')
    def validate(converter, val):
        if val is not None:
            if isinstance(val, str): pass
            elif isinstance(val, unicode): val = val.encode(converter.encoding)
            else: raise TypeError('Value type for attribute %s must be str in encoding %r. Got: %r'
                                  % (converter.attr, converter.encoding, type(val)))
        return BasestringConverter.validate(converter, val)
    def py2sql(converter, val):
        return val.decode(converter.encoding)
    def sql2py(converter, val):
        return val.encode(converter.encoding, 'replace')

class IntConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
        min_val = keyargs.pop('min', None)
        if min_val is not None and not isinstance(min_val, (int, long)):
            raise TypeError("'min' argument for attribute %s must be int. Got: %r" % (attr, min_val))
        max_val = keyargs.pop('max', None)
        if max_val is not None and not isinstance(max_val, (int, long)):
            raise TypeError("'max' argument for attribute %s must be int. Got: %r" % (attr, max_val))
        converter.min_val = min_val
        converter.max_val = max_val
    def validate(converter, val):
        if not isinstance(val, (int, long)):
            raise TypeError('Value type for attribute %s must be int. Got: %r' % (converter.attr, type(val)))
        if converter.min_val and val < converter.min_val:
            raise ValueError('Value %r of attr %s is less than the minimum allowed value %r'
                             % (val, converter.attr, converter.min_val))
        if converter.max_val and val > converter.max_val:
            raise ValueError('Value %r of attr %s is greater than the maximum allowed value %r'
                             % (val, converter.attr, converter.max_val))
        return val
    def sql_type(converter):
        return 'INTEGER'

class LongConverter(IntConverter):
    def sql_type(converter):
        return 'BIGINT'

class RealConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
        min_val = keyargs.pop('min', None)
        if min_val is not None:
            try: min_val = float(min_val)
            except ValueError:
                raise TypeError("Invalid value for 'min' argument for attribute %s: %r" % (attr, min_val))
        max_val = keyargs.pop('max', None)
        if max_val is not None:
            try: max_val = float(max_val)
            except ValueError:
                raise TypeError("Invalid value for 'max' argument for attribute %s: %r" % (attr, max_val))
        converter.min_val = min_val
        converter.max_val = max_val
    def validate(converter, val):
        try: val = float(val)
        except ValueError:
            raise TypeError('Invalid value for attribute %s: %r' % (converter.attr, val))
        if converter.min_val and val < converter.min_val:
            raise ValueError('Value %r of attr %s is less than the minimum allowed value %r'
                             % (val, converter.attr, converter.min_val))
        if converter.max_val and val > converter.max_val:
            raise ValueError('Value %r of attr %s is greater than the maximum allowed value %r'
                             % (val, converter.attr, converter.max_val))
        return val
    def sql_type(converter):
        return 'DOUBLE'

class DecimalConverter(Converter):
    def __init__(converter, attr=None):
        Converter.__init__(converter, attr)
    def init(converter, keyargs):
        attr = converter.attr
        args = attr.args
        if len(args) > 2: raise TypeError('Too many positional parameters for Decimal (expected: precision and scale)')

        if args: precision = args[0]
        else: precision = keyargs.pop('precision', 12)
        if not isinstance(precision, (int, long)):
            raise TypeError("'precision' positional argument for attribute %s must be int. Got: %r" % (attr, precision))
        if precision <= 0: raise TypeError(
            "'precision' positional argument for attribute %s must be positive. Got: %r" % (attr, precision))

        if len(args) == 2: scale = args[1]
        else: scale = keyargs.pop('scale', 2)
        if not isinstance(scale, (int, long)):
            raise TypeError("'scale' positional argument for attribute %s must be int. Got: %r" % (attr, scale))
        if scale <= 0: raise TypeError(
            "'scale' positional argument for attribute %s must be positive. Got: %r" % (attr, scale))

        if scale > precision: raise ValueError("'scale' must be less or equal 'precision'")
        converter.precision = precision
        converter.scale = scale
        converter.exp = Decimal(10) ** -scale

        min_val = keyargs.pop('min', None)
        if min_val is not None:
            try: min_val = Decimal(min_val)
            except TypeError: raise TypeError(
                "Invalid value for 'min' argument for attribute %s: %r" % (attr, min_val))

        max_val = keyargs.pop('max', None)
        if max_val is not None:
            try: max_val = Decimal(max_val)
            except TypeError: raise TypeError(
                "Invalid value for 'max' argument for attribute %s: %r" % (attr, max_val))

        converter.min_val = min_val
        converter.max_val = max_val
    def validate(converter, val):
        if type(val) is Decimal: return val
        try: return Decimal(val)
        except InvalidOperation, exc:
            raise TypeError('Invalid value for attribute %s: %r' % (converter.attr, val))
        if converter.min_val is not None and val < converter.min_val:
            raise ValueError('Value %r of attr %s is less than the minimum allowed value %r'
                             % (val, converter.attr, converter.min_val))
        if converter.max_val is not None and val > converter.max_val:
            raise ValueError('Value %r of attr %s is greater than the maximum allowed value %r'
                             % (val, converter.attr, converter.max_val))
    def sql_type(converter):
        return 'DECIMAL(%d, %d)' % (converter.precision, converter.scale)

class BlobConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
    def validate(converter, val):
        if isinstance(val, buffer): return val
        if isinstance(val, str): return buffer(val)
        raise TypeError("Attribute %r: expected type is 'buffer'. Got: %r" % (converter.attr, type(val)))
    def sql_type(converter):
        return 'LONGBLOB'

class DatetimeConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
    def validate(converter, val):
        if not isinstance(val, datetime):
            raise TypeError("Attribute %r: expected type is 'datetime'. Got: %r" % (converter.attr, val))
        return val
    def sql_type(converter):
        return 'DATETIME'

class DateConverter(Converter):
    def init(converter, keyargs):
        attr = converter.attr
        if attr and attr.args: unexpected_args(attr, attr.args)
    def validate(converter, val):
        if isinstance(val, datetime): return val.date()
        if not isinstance(val, date):
            raise TypeError("Attribute %r: expected type is 'date'. Got: %r" % (converter.attr, val))
        return val
    def sql_type(converter):
        return 'DATE'<|endoftext|>


<commit_before>import sqlite3 as sql
import sys
import os
import logging

# TODO:
# Need to verify correctness
# Use command-line args to pass in file names

# Files
txt_file = 'final100.txt'
opened_file = open(txt_file, 'U')
log_file = 't2db.log'

# Set Up SQL Connections
con = sql.connect('invpat_final_from_DVN.sqlite3') # Database to connect to
fin = sql.connect('final_test.sqlite3')  # Database to write to

# Logging
logging.basicConfig(filename=log_file, level=logging.DEBUG)
open(log_file, "w")

# Create Final table to be inserted into
with fin:
    fin_cur = fin.cursor()
    fin_cur.execute("DROP TABLE IF EXISTS Final;")

    # Schema for invpat
    #fin_cur.execute("""
    #            CREATE TABLE Final(
    #            Firstname TEXT,
    #            Lastname TEXT,
    #            Street TEXT,
    #            City TEXT,
    #            State TEXT,
    #            Country TEXT,
    #            Zipcode TEXT,
    #            Lat REAL,
    #            Lng REAL,
    #            InvSeq INT,
    #            Patent TEXT,
    #            AppYear INT,
    #            GYear INT,
    #            AppDate TEXT,
    #            Assignee TEXT,
    #            AsgNum INT,
    #            Class TEXT,
    #            Invnum TEXT,
    #            Invnum_N TEXT,
    #            Invnum_N_UC,
    #            Density REAL,
    #            Precision REAL,
    #            Recall REAL,
    #            Finalnum INT,
    #            Reldocs TEXT
    #            );
    #            """)

    # Schema for invpat_final
    fin_cur.execute("""
                CREATE TABLE Final(
                Firstname TEXT,
                Lastname TEXT,
                Street TEXT,
                City TEXT,
                State TEXT,
                Country TEXT,
                Zipcode TEXT,
                Lat REAL,
                Lon,
                InvSeq INT,
                Patent TEXT,
                GYear INT,
                AppYearStr TEXT,
                AppDateStr TEXT,
                Assignee TEXT,
                AsgNum INT,
                Class TEXT,
                Invnum,
                lower,
                upper,
                Finalnum INT,
                Reldocs TEXT
                );
                """)

# Match out of con, append, and join operation into fin
with con:
    con_cur = con.cursor()
    logging.info("Beginning to query database")
    count = 0
    success = 0
    errors = 0
    # con_cur.execute("CREATE INDEX index_patent ON invpat (Patent)");
    while True:
        line_read = opened_file.readline()
        if not line_read:
            break

        # Inv_Num ### Number ### Record-ID

        count = count + 1
        # print "count is...", count
        text_line = line_read.rstrip(',\n').split("###")
        inv_num = text_line[0]
        num = int(text_line[1]) # Convert str to int to be consistent.
        final_docs = text_line[2]
        final_docs_split = final_docs.split(',')
        # print inv_num, "###", num, "###", final_docs
        con_cur.execute("SELECT * FROM invpat WHERE (Invnum = \"%s\");" % inv_num)

        # con_cur.execute("SELECT * FROM invpat WHERE (Lastname
        # = \"FLEMING\" and Firstname = \"LEE\");") # Sanity Check

        fetched_value = con_cur.fetchone()  # Get match

        if fetched_value:
            success = success + 1
            value_to_insert = list(fetched_value)
            value_to_insert.append(num)
            value_to_insert.append(final_docs)
            fin_cur.execute("""INSERT INTO Final VALUES (?,?,?,?,?,?,?,?,?,
                            ?,?,?,?,?,?,?,?,?,?,?,?,?)""",
                            tuple(value_to_insert))
            if len(final_docs_split) > 1:
                for i, final_doc in enumerate(final_docs_split):
                    if i > 0:
                        rel_patent_number = final_doc.split("-")[0]
                        # print rel_patent_number
                        con_cur.execute("SELECT * FROM invpat WHERE Patent = \"%s\";" % rel_patent_number)
                        fetched_value = con_cur.fetchone()  # Get match
                        # print fetched_value
                        if fetched_value:
                            # print "not skipping rel pat"
                            value_to_insert = list(fetched_value)
                            value_to_insert.append(num)
                            value_to_insert.append(final_docs)
                            fin_cur.execute("""INSERT INTO Final VALUES (?,?,?,?,?,?,?,?,?,

                                            ?,?,?,?,?,?,?,?,?,?,?,?,?)""",
                                            tuple(value_to_insert))
                        else:
                            logging.error("Did not find a match for rel pat %s for invnum %s" % (rel_patent_number, inv_num))
        else:
            errors = errors + 1
            logging.error("Did not find a match for Invnum %s" % inv_num)

logging.info("Successfully did %d querys" % success)
logging.info("Failed %d querys" % errors)
logging.info("Total %d querys" % count)
fin.commit()

<commit_msg>added more logging for rel pats<commit_after>import sqlite3 as sql
import sys
import os
import logging

# TODO:
# Need to verify correctness
# Use command-line args to pass in file names

# Files
txt_file = 'final100.txt'
opened_file = open(txt_file, 'U')
log_file = 't2db.log'

# Set Up SQL Connections
con = sql.connect('invpat_final_from_DVN.sqlite3') # Database to connect to
fin = sql.connect('final_test.sqlite3')  # Database to write to

# Logging
logging.basicConfig(filename=log_file, level=logging.DEBUG)
open(log_file, "w")

# Create Final table to be inserted into
with fin:
    fin_cur = fin.cursor()
    fin_cur.execute("DROP TABLE IF EXISTS Final;")

    # Schema for invpat
    #fin_cur.execute("""
    #            CREATE TABLE Final(
    #            Firstname TEXT,
    #            Lastname TEXT,
    #            Street TEXT,
    #            City TEXT,
    #            State TEXT,
    #            Country TEXT,
    #            Zipcode TEXT,
    #            Lat REAL,
    #            Lng REAL,
    #            InvSeq INT,
    #            Patent TEXT,
    #            AppYear INT,
    #            GYear INT,
    #            AppDate TEXT,
    #            Assignee TEXT,
    #            AsgNum INT,
    #            Class TEXT,
    #            Invnum TEXT,
    #            Invnum_N TEXT,
    #            Invnum_N_UC,
    #            Density REAL,
    #            Precision REAL,
    #            Recall REAL,
    #            Finalnum INT,
    #            Reldocs TEXT
    #            );
    #            """)

    # Schema for invpat_final
    fin_cur.execute("""
                CREATE TABLE Final(
                Firstname TEXT,
                Lastname TEXT,
                Street TEXT,
                City TEXT,
                State TEXT,
                Country TEXT,
                Zipcode TEXT,
                Lat REAL,
                Lon,
                InvSeq INT,
                Patent TEXT,
                GYear INT,
                AppYearStr TEXT,
                AppDateStr TEXT,
                Assignee TEXT,
                AsgNum INT,
                Class TEXT,
                Invnum,
                lower,
                upper,
                Finalnum INT,
                Reldocs TEXT
                );
                """)

# Match out of con, append, and join operation into fin
with con:
    con_cur = con.cursor()
    logging.info("Beginning to query database")
    count = 0
    success = 0
    errors = 0
    relpats = 0
    relpaterrors = 0
    # con_cur.execute("CREATE INDEX index_patent ON invpat (Patent)");
    while True:
        line_read = opened_file.readline()
        if not line_read:
            break

        # Inv_Num ### Number ### Record-ID

        count = count + 1
        # print "count is...", count
        text_line = line_read.rstrip(',\n').split("###")
        inv_num = text_line[0]
        num = int(text_line[1]) # Convert str to int to be consistent.
        final_docs = text_line[2]
        final_docs_split = final_docs.split(',')
        # print inv_num, "###", num, "###", final_docs
        con_cur.execute("SELECT * FROM invpat WHERE (Invnum = \"%s\");" % inv_num)

        # con_cur.execute("SELECT * FROM invpat WHERE (Lastname
        # = \"FLEMING\" and Firstname = \"LEE\");") # Sanity Check

        fetched_value = con_cur.fetchone()  # Get match

        if fetched_value:
            success = success + 1
            value_to_insert = list(fetched_value)
            value_to_insert.append(num)
            value_to_insert.append(final_docs)
            fin_cur.execute("""INSERT INTO Final VALUES (?,?,?,?,?,?,?,?,?,
                            ?,?,?,?,?,?,?,?,?,?,?,?,?)""",
                            tuple(value_to_insert))
            if len(final_docs_split) > 1:
                for i, final_doc in enumerate(final_docs_split):
                    if i > 0:
                        rel_patent_number = final_doc.split("-")[0]
                        # print rel_patent_number
                        con_cur.execute("SELECT * FROM invpat WHERE Patent = \"%s\";" % rel_patent_number)
                        fetched_value = con_cur.fetchone()  # Get match
                        # print fetched_value
                        if fetched_value:
                            relpats = relpats + 1
                            # print "not skipping rel pat"
                            value_to_insert = list(fetched_value)
                            value_to_insert.append(num)
                            value_to_insert.append(final_docs)
                            fin_cur.execute("""INSERT INTO Final VALUES (?,?,?,?,?,?,?,?,?,

                                            ?,?,?,?,?,?,?,?,?,?,?,?,?)""",
                                            tuple(value_to_insert))
                        else:
                            relpaterrors = relpaterrors + 1
                            logging.error("Did not find a match for rel pat %s for invnum %s" % (rel_patent_number, inv_num))
        else:
            errors = errors + 1
            logging.error("Did not find a match for Invnum %s" % inv_num)

logging.info("Successfully did %d invnums" % success)
logging.info("Failed %d invnums" % errors)
logging.info("Successfully did %d relpats" % relpats)
logging.info("Failed %d relpats" % relpaterrors)
logging.info("Total %d querys" % count)
fin.commit()

<|endoftext|>


<commit_before># Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
# Copyright 2010 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Stubouts, mocks and fixtures for the test suite."""

import copy
import datetime

from nova import exception

class FakeModel(object):
    """Stubs out for model."""
    def __init__(self, values):
        self.values = values

    def __getattr__(self, name):
        return self.values[name]

    def __getitem__(self, key):
        if key in self.values:
            return self.values[key]
        else:
            raise NotImplementedError()

    def __repr__(self):
        return '<FakeModel: %s>' % self.values

    def get(self, name):
        return self.values[name]

def stub_out(test, funcs):
    """Set the stubs in mapping in the db api."""
    for module, func in funcs.items():
        test.stub_out(module, func)

fixed_ip_fields = {'id': 0,
                   'network_id': 0,
                   'address': '192.168.0.100',
                   'instance': False,
                   'instance_uuid': 'eb57d790-fc60-4119-a51a-f2b0913bdc93',
                   'allocated': False,
                   'virtual_interface_id': 0,
                   'virtual_interface': None,
                   'floating_ips': []}

network_fields = {'id': 0,
                  'cidr': '192.168.0.0/24',
                  'netmask': '255.255.255.0',
                  'cidr_v6': 'dead:beef::/64',
                  'netmask_v6': '64',
                  'project_id': 'fake',
                  'label': 'fake',
                  'gateway': '192.168.0.1',
                  'bridge': 'fa0',
                  'bridge_interface': 'fake_fa0',
                  'broadcast': '192.168.0.255',
                  'gateway_v6': 'dead:beef::1',
                  'dns': '192.168.0.1',
                  'vlan': None,
                  'host': None,
                  'injected': False,
                  'vpn_public_address': '192.168.0.2'}

flavor_fields = {'id': 0,
                 'rxtx_cap': 3}

floating_ip_fields = {'id': 0,
                      'address': '192.168.1.100',
                      'fixed_ip_id': None,
                      'fixed_ip': None,
                      'project_id': None,
                      'pool': 'nova',
                      'auto_assigned': False}

virtual_interface_fields = {'id': 0,
                            'address': 'DE:AD:BE:EF:00:00',
                            'network_id': 0,
                            'instance_id': 0,
                            'network': FakeModel(network_fields)}

fixed_ips = [fixed_ip_fields]
floating_ips = [floating_ip_fields]
virtual_interfaces = [virtual_interface_fields]
networks = [network_fields]

def fake_floating_ip_allocate_address(context, project_id, pool,
                                      auto_assigned=False):
    ips = [i for i in floating_ips if i['fixed_ip_id'] is None and
           i['project_id'] is None and i['pool'] == pool]
    if not ips:
        raise exception.NoMoreFloatingIps()
    ips[0]['project_id'] = project_id
    ips[0]['auto_assigned'] = auto_assigned
    return FakeModel(ips[0])

def fake_floating_ip_deallocate(context, address):
    ips = [i for i in floating_ips if i['address'] == address]
    if ips:
        ips[0]['project_id'] = None
        ips[0]['auto_assigned'] = False

def fake_floating_ip_disassociate(context, address):
    ips = [i for i in floating_ips if i['address'] == address]
    if ips:
        fixed_ip_address = None
        if ips[0]['fixed_ip']:
            fixed_ip_address = ips[0]['fixed_ip']['address']
        ips[0]['fixed_ip'] = None
        ips[0]['host'] = None
        return fixed_ip_address

def fake_floating_ip_fixed_ip_associate(context, floating_address,
                                        fixed_address, host):
    float = [i for i in floating_ips if i['address'] == floating_address]
    fixed = [i for i in fixed_ips if i['address'] == fixed_address]
    if float and fixed:
        float[0]['fixed_ip'] = fixed[0]
        float[0]['fixed_ip_id'] = fixed[0]['id']
        float[0]['host'] = host

def fake_floating_ip_get_all_by_host(context, host):
    # TODO(jkoelker): Once we get the patches that remove host from
    #                 the floating_ip table, we'll need to stub
    #                 this out
    pass

def fake_floating_ip_get_by_address(context, address):
    if isinstance(address, FakeModel):
        # NOTE(tr3buchet): yo dawg, i heard you like addresses
        address = address['address']
    ips = [i for i in floating_ips if i['address'] == address]
    if not ips:
        raise exception.FloatingIpNotFoundForAddress(address=address)
    return FakeModel(ips[0])

def fake_fixed_ip_associate(context, address, instance_id):
    ips = [i for i in fixed_ips if i['address'] == address]
    if not ips:
        raise exception.NoMoreFixedIps(net='fake_net')
    ips[0]['instance'] = True
    ips[0]['instance_id'] = instance_id

def fake_fixed_ip_associate_pool(context, network_id, instance_id):
    ips = [i for i in fixed_ips if not i['instance'] and
           (i['network_id'] == network_id or i['network_id'] is None)]
    if not ips:
        raise exception.NoMoreFixedIps(net=network_id)
    ips[0]['instance'] = True
    ips[0]['instance_id'] = instance_id
    return ips[0]['address']

def fake_fixed_ip_create(context, values):
    ip = dict(fixed_ip_fields)
    ip['id'] = max([i['id'] for i in fixed_ips] or [-1]) + 1
    for key in values:
        ip[key] = values[key]
    return ip

def fake_fixed_ip_disassociate(context, address):
    ips = [i for i in fixed_ips if i['address'] == address]
    if ips:
        ips[0]['instance_id'] = None
        ips[0]['instance'] = None
        ips[0]['virtual_interface'] = None
        ips[0]['virtual_interface_id'] = None

def fake_fixed_ip_disassociate_all_by_timeout(context, host, time):
    return 0

def fake_fixed_ip_get_all(context):
    return [FakeModel(i) for i in fixed_ips]

def fake_fixed_ip_get_by_instance(context, instance_uuid):
    ips = [i for i in fixed_ips if i['instance_uuid'] == instance_uuid]
    return [FakeModel(i) for i in ips]

def fake_fixed_ip_get_by_address(context, address):
    ips = [i for i in fixed_ips if i['address'] == address]
    if ips:
        return FakeModel(ips[0])

def fake_fixed_ip_update(context, address, values):
    ips = [i for i in fixed_ips if i['address'] == address]
    fif = copy.deepcopy(fixed_ip_fields)
    if ips:
        for key in values:
            ips[0][key] = values[key]
            if key == 'virtual_interface_id':
                vif = [v for v in virtual_interfaces
                       if v['id'] == values[key]]
                if not vif:
                    continue
                fif['virtual_interface'] = FakeModel(vif[0])

def fake_virtual_interface_create(context, values):
    vif = dict(virtual_interface_fields)
    vif['id'] = max([m['id'] for m in virtual_interfaces] or [-1]) + 1
    for key in values:
        vif[key] = values[key]
    return FakeModel(vif)

def fake_virtual_interface_delete_by_instance(context, instance_id):
    vif = copy.copy(virtual_interfaces)
    addresses = [m for m in vif
                 if m['instance_id'] == instance_id]
    try:
        for address in addresses:
            vif.remove(address)
    except ValueError:
        pass

def fake_virtual_interface_get_by_instance(context, instance_id):
    return [FakeModel(m) for m in virtual_interfaces
            if m['instance_id'] == instance_id]

def fake_virtual_interface_get_by_instance_and_network(context,
                                                       instance_id,
                                                       network_id):
    vif = [v for v in virtual_interfaces if v['instance_id'] == instance_id and
           v['network_id'] == network_id]
    if not vif:
        return None
    return FakeModel(vif[0])

def fake_network_create_safe(context, values):
    net = dict(network_fields)
    net['id'] = max([n['id'] for n in networks] or [-1]) + 1
    for key in values:
        net[key] = values[key]
    return FakeModel(net)

def fake_network_get(context, network_id):
    net = [n for n in networks if n['id'] == network_id]
    if not net:
        return None
    return FakeModel(net[0])

def fake_network_get_all(context):
    return [FakeModel(n) for n in networks]

def fake_network_get_all_by_host(context, host):
    nets = [n for n in networks if n['host'] == host]
    return [FakeModel(n) for n in nets]

def fake_network_set_host(context, network_id, host_id):
    nets = [n for n in networks if n['id'] == network_id]
    for net in nets:
        net['host'] = host_id
    return host_id

def fake_network_update(context, network_id, values):
    nets = [n for n in networks if n['id'] == network_id]
    for net in nets:
        for key in values:
            net[key] = values[key]

def fake_project_get_networks(context, project_id):
    return [FakeModel(n) for n in networks
            if n['project_id'] == project_id]

def stub_out_db_network_api(test):

    funcs = [fake_floating_ip_allocate_address,
             fake_floating_ip_deallocate,
             fake_floating_ip_disassociate,
             fake_floating_ip_fixed_ip_associate,
             fake_floating_ip_get_all_by_host,
             fake_floating_ip_get_by_address,
             fake_fixed_ip_associate,
             fake_fixed_ip_associate_pool,
             fake_fixed_ip_create,
             fake_fixed_ip_disassociate,
             fake_fixed_ip_disassociate_all_by_timeout,
             fake_fixed_ip_get_all,
             fake_fixed_ip_get_by_instance,
             fake_fixed_ip_get_by_address,
             fake_fixed_ip_update,
             fake_virtual_interface_create,
             fake_virtual_interface_delete_by_instance,
             fake_virtual_interface_get_by_instance,
             fake_virtual_interface_get_by_instance_and_network,
             fake_network_create_safe,
             fake_network_get,
             fake_network_get_all,
             fake_network_get_all_by_host,
             fake_network_set_host,
             fake_network_update,
             fake_project_get_networks]
    funcs = {'nova.db.api.%s' % fn.__name__.replace('fake_', ''): fn
             for fn in funcs}

    stub_out(test, funcs)

def stub_out_db_instance_api(test, injected=True):
    """Stubs out the db API for creating Instances."""

    def _create_instance_type(**updates):
        instance_type = {'id': 2,
                         'name': 'm1.tiny',
                         'memory_mb': 512,
                         'vcpus': 1,
                         'vcpu_weight': None,
                         'root_gb': 0,
                         'ephemeral_gb': 10,
                         'flavorid': 1,
                         'rxtx_factor': 1.0,
                         'swap': 0,
                         'deleted_at': None,
                         'created_at': datetime.datetime(2014, 8, 8, 0, 0, 0),
                         'updated_at': None,
                         'deleted': False,
                         'disabled': False,
                         'is_public': True,
                         'extra_specs': {},
                         'description': None
                        }
        if updates:
            instance_type.update(updates)
        return instance_type

    INSTANCE_TYPES = {
        'm1.tiny': _create_instance_type(
                        id=2,
                        name='m1.tiny',
                        memory_mb=512,
                        vcpus=1,
                        vcpu_weight=None,
                        root_gb=0,
                        ephemeral_gb=10,
                        flavorid=1,
                        rxtx_factor=1.0,
                        swap=0),
        'm1.small': _create_instance_type(
                        id=5,
                        name='m1.small',
                        memory_mb=2048,
                        vcpus=1,
                        vcpu_weight=None,
                        root_gb=20,
                        ephemeral_gb=0,
                        flavorid=2,
                        rxtx_factor=1.0,
                        swap=1024),
        'm1.medium': _create_instance_type(
                        id=1,
                         name='m1.medium',
                         memory_mb=4096,
                         vcpus=2,
                         vcpu_weight=None,
                         root_gb=40,
                         ephemeral_gb=40,
                         flavorid=3,
                         rxtx_factor=1.0,
                         swap=0),
        'm1.large': _create_instance_type(
                        id=3,
                         name='m1.large',
                         memory_mb=8192,
                         vcpus=4,
                         vcpu_weight=10,
                         root_gb=80,
                         ephemeral_gb=80,
                         flavorid=4,
                         rxtx_factor=1.0,
                         swap=0),
        'm1.xlarge': _create_instance_type(
                         id=4,
                         name='m1.xlarge',
                         memory_mb=16384,
                         vcpus=8,
                         vcpu_weight=None,
                         root_gb=160,
                         ephemeral_gb=160,
                         flavorid=5,
                         rxtx_factor=1.0,
                         swap=0)}

    fixed_ip_fields = {'address': '10.0.0.3',
                       'address_v6': 'fe80::a00:3',
                       'network_id': 'fake_flat'}

    def fake_flavor_get_all(*a, **k):
        return INSTANCE_TYPES.values()

    @classmethod
    def fake_flavor_get_by_name(cls, context, name):
        return INSTANCE_TYPES[name]

    @classmethod
    def fake_flavor_get(cls, context, id):
        for inst_type in INSTANCE_TYPES.values():
            if str(inst_type['id']) == str(id):
                return inst_type
        return None

    def fake_fixed_ip_get_by_instance(context, instance_id):
        return [FakeModel(fixed_ip_fields)]

    funcs = {
        'nova.objects.flavor._flavor_get_all_from_db': (
            fake_flavor_get_all),
        'nova.objects.Flavor._flavor_get_by_name_from_db': (
            fake_flavor_get_by_name),
        'nova.objects.Flavor._flavor_get_from_db': fake_flavor_get,
        'nova.db.api.fixed_ip_get_by_instance': fake_fixed_ip_get_by_instance,
    }
    stub_out(test, funcs)
<commit_msg>nova-net: Remove unused 'stub_out_db_network_api'<commit_after># Copyright (c) 2011 X.commerce, a business unit of eBay Inc.
# Copyright 2010 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Stubouts, mocks and fixtures for the test suite."""

import datetime

def stub_out(test, funcs):
    """Set the stubs in mapping in the db api."""
    for module, func in funcs.items():
        test.stub_out(module, func)

def stub_out_db_instance_api(test, injected=True):
    """Stubs out the db API for creating Instances."""

    def _create_instance_type(**updates):
        instance_type = {'id': 2,
                         'name': 'm1.tiny',
                         'memory_mb': 512,
                         'vcpus': 1,
                         'vcpu_weight': None,
                         'root_gb': 0,
                         'ephemeral_gb': 10,
                         'flavorid': 1,
                         'rxtx_factor': 1.0,
                         'swap': 0,
                         'deleted_at': None,
                         'created_at': datetime.datetime(2014, 8, 8, 0, 0, 0),
                         'updated_at': None,
                         'deleted': False,
                         'disabled': False,
                         'is_public': True,
                         'extra_specs': {},
                         'description': None
                        }
        if updates:
            instance_type.update(updates)
        return instance_type

    INSTANCE_TYPES = {
        'm1.tiny': _create_instance_type(
                        id=2,
                        name='m1.tiny',
                        memory_mb=512,
                        vcpus=1,
                        vcpu_weight=None,
                        root_gb=0,
                        ephemeral_gb=10,
                        flavorid=1,
                        rxtx_factor=1.0,
                        swap=0),
        'm1.small': _create_instance_type(
                        id=5,
                        name='m1.small',
                        memory_mb=2048,
                        vcpus=1,
                        vcpu_weight=None,
                        root_gb=20,
                        ephemeral_gb=0,
                        flavorid=2,
                        rxtx_factor=1.0,
                        swap=1024),
        'm1.medium': _create_instance_type(
                        id=1,
                         name='m1.medium',
                         memory_mb=4096,
                         vcpus=2,
                         vcpu_weight=None,
                         root_gb=40,
                         ephemeral_gb=40,
                         flavorid=3,
                         rxtx_factor=1.0,
                         swap=0),
        'm1.large': _create_instance_type(
                        id=3,
                         name='m1.large',
                         memory_mb=8192,
                         vcpus=4,
                         vcpu_weight=10,
                         root_gb=80,
                         ephemeral_gb=80,
                         flavorid=4,
                         rxtx_factor=1.0,
                         swap=0),
        'm1.xlarge': _create_instance_type(
                         id=4,
                         name='m1.xlarge',
                         memory_mb=16384,
                         vcpus=8,
                         vcpu_weight=None,
                         root_gb=160,
                         ephemeral_gb=160,
                         flavorid=5,
                         rxtx_factor=1.0,
                         swap=0)}

    def fake_flavor_get_all(*a, **k):
        return INSTANCE_TYPES.values()

    @classmethod
    def fake_flavor_get_by_name(cls, context, name):
        return INSTANCE_TYPES[name]

    @classmethod
    def fake_flavor_get(cls, context, id):
        for inst_type in INSTANCE_TYPES.values():
            if str(inst_type['id']) == str(id):
                return inst_type
        return None

    funcs = {
        'nova.objects.flavor._flavor_get_all_from_db': (
            fake_flavor_get_all),
        'nova.objects.Flavor._flavor_get_by_name_from_db': (
            fake_flavor_get_by_name),
        'nova.objects.Flavor._flavor_get_from_db': fake_flavor_get,
    }
    stub_out(test, funcs)
<|endoftext|>