<commit_before>from Drawable import Drawable
from Prop import Prop
from image import AsyncImageLoad
from anim import AnimManager
from anim.AnimDriver import IterableAnimDriver
from pygame import Rect
from pygame import transform
from pygame import draw as Draw
from pygame import Color

from maths.Vect2 import Vect2
from maths import Ease

class MultiPhotoView(Drawable):
    SCROLL_SPEED = 1.0
    DISPLAY_SPEED = 3.0

    # a rectangular sprite that displays a photo on screen for a certain amount of time
    class PhotoProp(Prop):
        STATE_SCROLL_IN, STATE_DISPLAY, STATE_SCROLL_OUT, NUM_STATES = range(4)
        def __init__(self, images, w, h, state_listener):
            super(MultiPhotoView.PhotoProp, self).__init__()

            self.state_listeners = []
            if state_listener:
                self.state_listeners.append(state_listener)

                scale = min( float(MultiPhotoView.rect.w) / float(w), float(MultiPhotoView.rect.h) / float(h))
                new_w = int(float(w) * scale)
                new_h = int(float(h) * scale)

                self.frames = []
                for frame in images:
                    self.frames.append(transform.smoothscale(frame, (new_w, new_h)))

                self.rect = Rect(0, 0, new_w, new_h)
                self.pos = Vect2((0, -new_h))
            else:
                self.frames = images
                self.rect = Rect(0, 0, w, h)
                self.pos = Vect2((0, -w))

            self.display_time = 0
            self.done = False

            self._state = None

        @property
        def state(self):
            return self._state

        @state.setter
        def state(self, new_state):
            self._state = new_state
            for listener in self.state_listeners:
                listener.onPhotoPropStateChange(self, new_state)

        # def draw(self, display_screen, dT=None):
        #     super(MultiPhotoView.PhotoProp, self).draw(display_screen, dT)
        #     if self.rect:
        #         dbg_rect = Rect(self.pos[0], self.pos[1], self.rect.w, self.rect.h)
        #         Draw.rect(display_screen, Color("red"), dbg_rect, 1)

        def show(self, scroll_speed, display_speed):
            self.scroll_speed = scroll_speed

            self.state = MultiPhotoView.PhotoProp.STATE_SCROLL_IN

            x = (MultiPhotoView.rect.w - self.rect.w) * 0.5
            y = (MultiPhotoView.rect.h - self.rect.h) * 0.5
            self.pos.x = x
            self.current_move_action = self.moveTo(x, y, self.scroll_speed, Ease.inOutCubic)
            AnimManager.addDriver(self.current_move_action)

            self.current_anim_action = IterableAnimDriver(self.frames, self, 'image', True, Ease.linear)
            AnimManager.addDriver(self.current_anim_action)

        def hide(self):
            self.state = MultiPhotoView.PhotoProp.STATE_SCROLL_OUT
            self.current_move_action = self.moveTo(self.pos.x, MultiPhotoView.rect.h, self.scroll_speed, Ease.inOutCubic)
            AnimManager.addDriver(self.current_move_action)

        def update(self, dT):
            if self.state == MultiPhotoView.PhotoProp.STATE_SCROLL_IN:
                if self.current_move_action.done:
                    self.state = MultiPhotoView.PhotoProp.STATE_DISPLAY
            elif self.state == MultiPhotoView.PhotoProp.STATE_SCROLL_OUT:
                if self.current_move_action.done:
                    self.current_move_action = None
                    self.done = True
                    self.current_anim_action.kill()
                    self.state_listeners = []
                if self.current_anim_action.done:
                    self.current_anim_action = None

    def __init__(self, blogModel, viewable_area):
        MultiPhotoView.rect = Rect(0, 0, viewable_area[0], viewable_area[1])
        self.model = blogModel
        self.post = 0
        # while the view isn't ready, the controller will not
        # ask it to increment posts. So during an image load
        # we will hold on the current post until the batch is done
        # processing in
        self.ready = False
        self.initialized = False
        # we  will hold on to ref of a photo prop that has just been
        # created, and once it reaches display state we trigger a preload
        self.preload_after_prop_displays = None

        self.img_queue = []
        self.last_cached_post_idx = self.post

        self.photo_props = []
        self.preload_images(5)

    def isReady(self):
        return self.ready

    def preload_images(self, look_ahead = 0):
        self.ready = False
        def callback(images, w, h):
            self.img_queue.append((images, w, h))
            if not self.initialized:
                self.initialized = True
                self.setPost(self.post)
        def lastInBatchCallback(images, w, h):
            callback(images, w, h)
            self.ready = True

        start, end = self.last_cached_post_idx, self.last_cached_post_idx + look_ahead
        posts, self.last_cached_post_idx = self.model.getPosts(self.last_cached_post_idx, self.last_cached_post_idx + look_ahead)
        #print "requested " + str(len(posts)) + " posts"

        # load each image in the batch. On the last image of the batch,
        # set the view's ready flag to true
        num_photos = sum(map(lambda x: x.getNumPhotos(), posts))
        photo_num = 0
        for i in range(len(posts)):
            for url in posts[i].getPhotos(None, self.rect.h):
                fCallback = callback
                if photo_num == num_photos - 1:
                    fCallback = lastInBatchCallback
                AsyncImageLoad.load(url, fCallback)
                photo_num += 1

        self.last_cached_post_idx += look_ahead + 1

    def preload_needed(self):
        return not self.preload_after_prop_displays and len(self.img_queue) < 5

    def incPost(self):
        self.setPost(self.post + 1)

    def setPost(self, idx):
        for prop in self.photo_props:
            prop.hide()

        images, w, h = self.img_queue.pop(0)

        self.post = idx

        photo_prop = MultiPhotoView.PhotoProp(images, w, h, self)
        if self.preload_needed():
            self.preload_after_prop_displays = photo_prop

        photo_prop.show(MultiPhotoView.SCROLL_SPEED, MultiPhotoView.DISPLAY_SPEED)
        self.photo_props.append(photo_prop)

    def onPhotoPropStateChange(self, photo_prop, state):
        if photo_prop == self.preload_after_prop_displays and state == MultiPhotoView.PhotoProp.STATE_DISPLAY:
            self.preload_images(5)
            self.preload_after_prop_displays = None

    def draw(self, display_screen, dT):
        not_done = []
        for prop in self.photo_props:
            prop.update(dT)
            if not prop.done:
                prop.draw(display_screen)
                not_done.append(prop)

        self.photo_props = not_done
    <commit_msg>cleaning up photo prop removal<commit_after>from Drawable import Drawable
from Prop import Prop
from image import AsyncImageLoad
from anim import AnimManager
from anim.AnimDriver import IterableAnimDriver
from pygame import Rect
from pygame import transform
from pygame import draw as Draw
from pygame import Color

from maths.Vect2 import Vect2
from maths import Ease

class MultiPhotoView(Drawable):
    SCROLL_SPEED = 1.0
    DISPLAY_SPEED = 3.0

    # a rectangular sprite that displays a photo on screen for a certain amount of time
    class PhotoProp(Prop):
        STATE_SCROLL_IN, STATE_DISPLAY, STATE_SCROLL_OUT, STATE_DEAD, NUM_STATES = range(5)
        def __init__(self, images, w, h, state_listener):
            super(MultiPhotoView.PhotoProp, self).__init__()

            self.state_listeners = []
            if state_listener:
                self.state_listeners.append(state_listener)

                scale = min( float(MultiPhotoView.rect.w) / float(w), float(MultiPhotoView.rect.h) / float(h))
                new_w = int(float(w) * scale)
                new_h = int(float(h) * scale)

                self.frames = []
                for frame in images:
                    self.frames.append(transform.smoothscale(frame, (new_w, new_h)))

                self.rect = Rect(0, 0, new_w, new_h)
                self.pos = Vect2((0, -new_h))
            else:
                self.frames = images
                self.rect = Rect(0, 0, w, h)
                self.pos = Vect2((0, -w))

            self.display_time = 0

            self._state = None

        @property
        def state(self):
            return self._state

        @state.setter
        def state(self, new_state):
            self._state = new_state
            for listener in self.state_listeners:
                listener.onPhotoPropStateChange(self, new_state)

        # def draw(self, display_screen, dT=None):
        #     super(MultiPhotoView.PhotoProp, self).draw(display_screen, dT)
        #     if self.rect:
        #         dbg_rect = Rect(self.pos[0], self.pos[1], self.rect.w, self.rect.h)
        #         Draw.rect(display_screen, Color("red"), dbg_rect, 1)

        def show(self, scroll_speed, display_speed):
            self.scroll_speed = scroll_speed

            self.state = MultiPhotoView.PhotoProp.STATE_SCROLL_IN

            x = (MultiPhotoView.rect.w - self.rect.w) * 0.5
            y = (MultiPhotoView.rect.h - self.rect.h) * 0.5
            self.pos.x = x
            self.current_move_action = self.moveTo(x, y, self.scroll_speed, Ease.inOutCubic)
            AnimManager.addDriver(self.current_move_action)

            self.current_anim_action = IterableAnimDriver(self.frames, self, 'image', True, Ease.linear)
            AnimManager.addDriver(self.current_anim_action)

        def hide(self):
            self.state = MultiPhotoView.PhotoProp.STATE_SCROLL_OUT
            self.current_move_action = self.moveTo(self.pos.x, MultiPhotoView.rect.h, self.scroll_speed, Ease.inOutCubic)
            AnimManager.addDriver(self.current_move_action)

        def update(self, dT):
            if self.state == MultiPhotoView.PhotoProp.STATE_SCROLL_IN:
                if self.current_move_action.done:
                    self.state = MultiPhotoView.PhotoProp.STATE_DISPLAY
            elif self.state == MultiPhotoView.PhotoProp.STATE_SCROLL_OUT:
                if self.current_move_action.done:
                    self.current_anim_action.kill()
                if self.current_anim_action.done:
                    self.state = MultiPhotoView.PhotoProp.STATE_DEAD

        def onRemove(self):
            self.image = None
            self.frames = []
            self.state_listeners = []
            self.current_move_action = None
            self.current_anim_action = None

    def __init__(self, blogModel, viewable_area):
        MultiPhotoView.rect = Rect(0, 0, viewable_area[0], viewable_area[1])
        self.model = blogModel
        self.post = 0
        # while the view isn't ready, the controller will not
        # ask it to increment posts. So during an image load
        # we will hold on the current post until the batch is done
        # processing in
        self.ready = False
        self.initialized = False
        # we  will hold on to ref of a photo prop that has just been
        # created, and once it reaches display state we trigger a preload
        self.preload_after_prop_displays = None

        self.img_queue = []
        self.last_cached_post_idx = self.post

        self.photo_props = []
        self.preload_images(5)

    def isReady(self):
        return self.ready

    def preload_images(self, look_ahead = 0):
        self.ready = False
        def callback(images, w, h):
            self.img_queue.append((images, w, h))
            if not self.initialized:
                self.initialized = True
                self.setPost(self.post)
        def lastInBatchCallback(images, w, h):
            callback(images, w, h)
            self.ready = True

        start, end = self.last_cached_post_idx, self.last_cached_post_idx + look_ahead
        posts, self.last_cached_post_idx = self.model.getPosts(self.last_cached_post_idx, self.last_cached_post_idx + look_ahead)
        #print "requested " + str(len(posts)) + " posts"

        # load each image in the batch. On the last image of the batch,
        # set the view's ready flag to true
        num_photos = sum(map(lambda x: x.getNumPhotos(), posts))
        photo_num = 0
        for i in range(len(posts)):
            for url in posts[i].getPhotos(None, self.rect.h):
                fCallback = callback
                if photo_num == num_photos - 1:
                    fCallback = lastInBatchCallback
                AsyncImageLoad.load(url, fCallback)
                photo_num += 1

        self.last_cached_post_idx += look_ahead + 1

    def preload_needed(self):
        return not self.preload_after_prop_displays and len(self.img_queue) < 5

    def incPost(self):
        self.setPost(self.post + 1)

    def setPost(self, idx):
        for prop in self.photo_props:
            prop.hide()

        images, w, h = self.img_queue.pop(0)

        self.post = idx

        photo_prop = MultiPhotoView.PhotoProp(images, w, h, self)
        if self.preload_needed():
            self.preload_after_prop_displays = photo_prop

        photo_prop.show(MultiPhotoView.SCROLL_SPEED, MultiPhotoView.DISPLAY_SPEED)
        self.photo_props.append(photo_prop)

    def onPhotoPropStateChange(self, photo_prop, state):
        if photo_prop == self.preload_after_prop_displays and state == MultiPhotoView.PhotoProp.STATE_DISPLAY:
            self.preload_images(5)
            self.preload_after_prop_displays = None
        elif state == MultiPhotoView.PhotoProp.STATE_DEAD:
            photo_prop.onRemove()
            self.photo_props.remove(photo_prop)

    def draw(self, display_screen, dT):
        for prop in self.photo_props:
            prop.update(dT)
            prop.draw(display_screen)
    <|endoftext|>


<commit_before>#!/usr/bin/env python3

import sys
import json
import os

from urllib.request import urlopen

import hashlib

from datetime import datetime

DEBUG = 0

def log_print(output):
    if DEBUG:
        print(output)

def gen_md5(str):
    md5str = hashlib.md5(str.encode(encoding='utf-8'))
    return md5str.hexdigest()

if __name__ == '__main__':
    # usage:
    # newpra.py xxx.json ppa-baseurl [rpaname]
    # newrpa.py result.json http://pools.corp.deepin.com/ppa/debian0311/
    # [e21b073b0d7d63d4b53c65b235309f37]

    if len(sys.argv) > 1:
        # get file list to download
        resultjson = sys.argv[1]
        baseurl = sys.argv[2]

        with open(resultjson, 'r') as f:
            data = json.load(f)

        modifytime = data['time']

        # example:
        # {'repo': 'debian', 'filelist':
        # ['pool/non-free/e/ebook-dev-alp/ebook-dev-alp_200407-2.dsc',
        # 'pool/non-free/e/ebook-dev-alp/ebook-dev-alp_200407.orig.tar.gz',
        # 'pool/non-free/e/ebook-dev-alp/ebook-dev-alp_200407-2.diff.gz'],
        # 'name': 'ebook-dev-alp', 'newversion': '200407-1', 'type': 'update',
        # 'arch': 'source', 'oldversion': '200407-2'}
        jsondetails = data['details']

        # sometimes not any packages updated
        if not jsondetails:
            print("There is no packages updated in this checkupdate, exit.")
            os._exit(1)

        # download all files
        if len(sys.argv) == 4:
            rpaname = sys.argv[3]
        else:
            rpaname = gen_md5(
                baseurl + datetime.now().strftime("%Y-%m-%d~%H%M%S"))

        rpapath = "/tmp/" + rpaname
        TMPDIR = "/tmp/rpa-" + rpaname

        log_print(TMPDIR)

        os.mkdir(TMPDIR)
        os.chdir(TMPDIR)
        os.mkdir("src")
        os.mkdir("deb")

        for fileitem in jsondetails:
            if fileitem['arch'] == 'source':
                for x in fileitem['filelist']:
                    fileurl = baseurl + "/" + x
                    log_print(fileurl)
                    g = urlopen(fileurl)
                    with open("src/" + os.path.basename(x), 'b+w') as f:
                        f.write(g.read())
            else:
                for x in fileitem['filelist']:
                    fileurl = baseurl + "/" + x
                    log_print(fileurl)
                    g = urlopen(fileurl)
                    with open("deb/" + os.path.basename(x), 'b+w') as f:
                        f.write(g.read())

        # make a new rpa from template
        os.system("cp -r /mnt/mirror-snapshot/utils/rpabase " + rpapath)
        # os.system("cp -r /tmp/rpabase " + rpapath)
        os.chdir(rpapath)
        # includedsc only one .dsc file each time
        dsccmd = 'find ' + TMPDIR + '/src/ -name "*.dsc" -exec reprepro -b ' + \
            rpapath + ' includedsc unstable {} \; >/dev/null 2>&1'
        os.system(dsccmd)
        os.system("reprepro -b . includedeb unstable " +
                  TMPDIR + "/deb/*.deb >/dev/null 2>&1")

        # to rpa
        basedir = "/srv/pool/base/rpa/" + rpaname
        wwwdir = "/srv/pool/www/rpa/" + rpaname
        # basedir = "/tmp/base/rpa/" + rpaname
        # wwwdir = "/tmp/www/rpa/" + rpaname

        if len(sys.argv) == 4:
            # update rpa has dirs, remove
            os.system("rm -rf " + basedir)
            os.system("rm -rf " + wwwdir)

        os.mkdir(basedir)
        os.mkdir(wwwdir)

        os.system("cp -rf " + rpapath + "/db " + basedir + "/")
        os.system("cp -rf " + rpapath + "/conf " + basedir + "/")

        os.system("cp -rf " + rpapath + "/dists " + wwwdir + "/")
        os.system("cp -rf " + rpapath + "/pool " + wwwdir + "/")
        os.system("cp -rf " + rpapath + "/checkupdate " + wwwdir + "/")

        # output the result json
        with open(wwwdir + '/checkupdate/result.json', 'w') as f:
            json.dump(data, f)

        # clean
        os.system("rm -rf " + TMPDIR)
        os.system("rm -rf " + rpapath)

        # end, print the rpaname
        print(rpaname)
<commit_msg>Add rpa size in result.json<commit_after>#!/usr/bin/env python3

import sys
import json
import os

from urllib.request import urlopen

import hashlib

from datetime import datetime

DEBUG = 0

def log_print(output):
    if DEBUG:
        print(output)

def gen_md5(str):
    md5str = hashlib.md5(str.encode(encoding='utf-8'))
    return md5str.hexdigest()

if __name__ == '__main__':
    # usage:
    # newpra.py xxx.json ppa-baseurl [rpaname]
    # newrpa.py result.json http://pools.corp.deepin.com/ppa/debian0311/
    # [e21b073b0d7d63d4b53c65b235309f37]

    if len(sys.argv) > 1:
        # get file list to download
        resultjson = sys.argv[1]
        baseurl = sys.argv[2]

        with open(resultjson, 'r') as f:
            data = json.load(f)

        modifytime = data['time']

        # example:
        # {'repo': 'debian', 'filelist':
        # ['pool/non-free/e/ebook-dev-alp/ebook-dev-alp_200407-2.dsc',
        # 'pool/non-free/e/ebook-dev-alp/ebook-dev-alp_200407.orig.tar.gz',
        # 'pool/non-free/e/ebook-dev-alp/ebook-dev-alp_200407-2.diff.gz'],
        # 'name': 'ebook-dev-alp', 'newversion': '200407-1', 'type': 'update',
        # 'arch': 'source', 'oldversion': '200407-2'}
        jsondetails = data['details']

        # sometimes not any packages updated
        if not jsondetails:
            print("There is no packages updated in this checkupdate, exit.")
            os._exit(1)

        # download all files
        if len(sys.argv) == 4:
            rpaname = sys.argv[3]
        else:
            rpaname = gen_md5(
                baseurl + datetime.now().strftime("%Y-%m-%d~%H%M%S"))

        rpapath = "/tmp/" + rpaname
        TMPDIR = "/tmp/rpa-" + rpaname

        log_print(TMPDIR)

        os.mkdir(TMPDIR)
        os.chdir(TMPDIR)
        os.mkdir("src")
        os.mkdir("deb")

        for fileitem in jsondetails:
            if fileitem['arch'] == 'source':
                for x in fileitem['filelist']:
                    fileurl = baseurl + "/" + x
                    log_print(fileurl)
                    g = urlopen(fileurl)
                    with open("src/" + os.path.basename(x), 'b+w') as f:
                        f.write(g.read())
            else:
                for x in fileitem['filelist']:
                    fileurl = baseurl + "/" + x
                    log_print(fileurl)
                    g = urlopen(fileurl)
                    with open("deb/" + os.path.basename(x), 'b+w') as f:
                        f.write(g.read())

        # make a new rpa from template
        os.system("cp -r /mnt/mirror-snapshot/utils/rpabase " + rpapath)
        # os.system("cp -r /tmp/rpabase " + rpapath)
        os.chdir(rpapath)
        # includedsc only one .dsc file each time
        dsccmd = 'find ' + TMPDIR + '/src/ -name "*.dsc" -exec reprepro -b ' + \
            rpapath + ' includedsc unstable {} \; >/dev/null 2>&1'
        os.system(dsccmd)
        os.system("reprepro -b . includedeb unstable " +
                  TMPDIR + "/deb/*.deb >/dev/null 2>&1")

        # to rpa
        basedir = "/srv/pool/base/rpa/" + rpaname
        wwwdir = "/srv/pool/www/rpa/" + rpaname
        # basedir = "/tmp/base/rpa/" + rpaname
        # wwwdir = "/tmp/www/rpa/" + rpaname

        if len(sys.argv) == 4:
            # update rpa has dirs, remove
            os.system("rm -rf " + basedir)
            os.system("rm -rf " + wwwdir)

        os.mkdir(basedir)
        os.mkdir(wwwdir)

        os.system("cp -rf " + rpapath + "/db " + basedir + "/")
        os.system("cp -rf " + rpapath + "/conf " + basedir + "/")

        os.system("cp -rf " + rpapath + "/dists " + wwwdir + "/")
        os.system("cp -rf " + rpapath + "/pool " + wwwdir + "/")
        os.system("cp -rf " + rpapath + "/checkupdate " + wwwdir + "/")

        # get the rpa size
        sizecmd = "du -sh " + TMPDIR + " 2>/dev/null | awk '{print $1;}'"
        rpasize = os.popen(sizecmd).read()
        if rpasize:
            data['size'] = rpasize

        # output the result json
        with open(wwwdir + '/checkupdate/result.json', 'w') as f:
            json.dump(data, f)

        # clean
        os.system("rm -rf " + TMPDIR)
        os.system("rm -rf " + rpapath)

        # end, print the rpaname
        print(rpaname)
<|endoftext|>


<commit_before># GStreamer Daemon - gst-launch on steroids
# Python client library abstracting gstd interprocess communication

# Copyright (c) 2015-2020 RidgeRun, LLC (http://www.ridgerun.com)

# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:

# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.

"""
GSTC - GstdError Class
"""

class GstdError(Exception):

    """Raised when Gstd IPC fails"""

    pass

"""
GSTC - GstcError Class
"""

class GstcError(Exception):

    """Raised when the Gstd python client fails internally"""

    pass
<commit_msg>Install description for GstcError and GstdError<commit_after># GStreamer Daemon - gst-launch on steroids
# Python client library abstracting gstd interprocess communication

# Copyright (c) 2015-2020 RidgeRun, LLC (http://www.ridgerun.com)

# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:

# 1. Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.

# 2. Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
# INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
# OF THE POSSIBILITY OF SUCH DAMAGE.

"""
GSTC - GstdError Class
"""

class GstdError(Exception):
    """Raised when Gstd IPC fails

    Attributes:
        description -- GstcError description message
    """

    def __init__(self, description):
        if description is None:
            description = "Gstd IPC failed"
        super(GstdError, self).__init__(description)
        self.description = description

"""
GSTC - GstcError Class
"""

class GstcError(Exception):
    """Raised when the Gstd python client fails internally

    Attributes:
        description -- GstcError description message
    """

    def __init__(self, description):
        if description is None:
            description = "Gstd Python client failed internally"
        super(GstcError, self).__init__(GstdError, description)
        self.description = description
<|endoftext|>


<commit_before>from .. import bar, hook, utils, manager
import base

class _GroupBase(base._Widget):
    def __init__(self, **config):
        base._Widget.__init__(self, bar.CALCULATED, **config)

    @property
    def fontsize(self):
        if self._fontsize is None:
            calc = self.bar.height - self.margin_y*2 - self.borderwidth*2 - self.padding*2
            return max(calc, 1)
        else:
            return self._fontsize

    @fontsize.setter
    def fontsize(self, value):
        self._fontsize = value

    def box_width(self):
        width, height = self.drawer.max_layout_size(
            [i.name for i in self.qtile.groups],
            self.font,
            self.fontsize
        )
        return width + self.padding*2 + self.margin_x*2 + self.borderwidth*2

    def _configure(self, qtile, bar):
        base._Widget._configure(self, qtile, bar)
        self.layout = self.drawer.textlayout("", "ffffff", self.font, self.fontsize)
        self.setup_hooks()

    def setup_hooks(self):
        def hook_response(*args, **kwargs):
            self.bar.draw()
        hook.subscribe.client_managed(hook_response)
        hook.subscribe.client_urgent_hint_changed(hook_response)
        hook.subscribe.client_killed(hook_response)
        hook.subscribe.setgroup(hook_response)
        hook.subscribe.group_window_add(hook_response)

    def drawbox(self, offset, text, bordercolor, textcolor, width=None):
        self.layout.text = text
        self.layout.font_family = self.font
        self.layout.font_size = self.fontsize
        self.layout.colour = textcolor
        if width is not None:
            self.layout.width = width
        framed = self.layout.framed(self.borderwidth, bordercolor, self.padding, self.padding)
        framed.draw(offset, self.margin_y)

class AGroupBox(_GroupBase):
    """
        A widget that graphically displays the current group.
    """
    defaults = manager.Defaults(
        ("margin_y", 3, "Y margin outside the box"),
        ("margin_x", 3, "X margin outside the box"),
        ("borderwidth", 3, "Current group border width"),
        ("font", "Arial", "Font face"),
        ("fontsize", None, "Font pixel size - calculated if None"),
        ("foreground", "aaaaaa", "Font colour"),
        ("background", "000000", "Widget background"),
        ("border", "215578", "Border colour"),
        ("padding", 5, "Padding inside the box")
    )
    def click(self, x, y, button):
        self.bar.screen.group.cmd_nextgroup()

    def calculate_width(self):
        return self.box_width()

    def draw(self):
        self.drawer.clear(self.background)
        e = (i for i in self.qtile.groups if i.name == self.bar.screen.group.name ).next()
        self.drawbox(self.margin_x, e.name, self.border, self.foreground)
        self.drawer.draw(self.offset, self.width)

class GroupBox(_GroupBase):
    """
        A widget that graphically displays the current group.
    """
    defaults = manager.Defaults(
        ("active", "FFFFFF", "Active group font colour"),
        ("inactive", "404040", "Inactive group font colour"),
        ("margin_y", 3, "Y margin outside the box"),
        ("margin_x", 3, "X margin outside the box"),
        ("borderwidth", 3, "Current group border width"),
        ("font", "Arial", "Font face"),
        ("fontsize", None, "Font pixel size - calculated if None"),
        ("background", "000000", "Widget background"),
        ("this_screen_border", "215578", "Border colour for group on this screen."),
        ("other_screen_border", "404040", "Border colour for group on other screen."),
        ("padding", 5, "Padding inside the box"),
        ("urgent_border", "FF0000", "Urgent border color")
    )
    def __init__(self, **config):
        base._Widget.__init__(self, bar.CALCULATED, **config)

    def click(self, x, y, button):
        groupOffset = int(x/self.box_width())
        if len(self.qtile.groups) - 1 >= groupOffset:
            self.bar.screen.setGroup(self.qtile.groups[groupOffset])

    def calculate_width(self):
        return self.box_width() * len(self.qtile.groups)

    def group_has_urgent(self, group):
        return len([w for w in group.windows if w.urgent]) > 0

    def draw(self):
        bw = self.box_width()
        self.drawer.clear(self.background)
        for i, g in enumerate(self.qtile.groups):
            if g.screen:
                if self.bar.screen.group.name == g.name:
                    border = self.this_screen_border
                else:
                    border = self.other_screen_border
            elif self.group_has_urgent(g):
                border = self.urgent_border
            else:
                border = self.background

            if g.windows:
                text = self.active
            else:
                text = self.inactive

            self.drawbox(
                self.margin_x + (bw*i),
                g.name,
                border,
                text,
                bw - self.margin_x*2 - self.padding*2
            )
        self.drawer.draw(self.offset, self.width)

<commit_msg>added scrolling support in groupbox<commit_after>from .. import bar, hook, utils, manager
import base

class _GroupBase(base._Widget):
    def __init__(self, **config):
        base._Widget.__init__(self, bar.CALCULATED, **config)

    @property
    def fontsize(self):
        if self._fontsize is None:
            calc = self.bar.height - self.margin_y*2 - self.borderwidth*2 - self.padding*2
            return max(calc, 1)
        else:
            return self._fontsize

    @fontsize.setter
    def fontsize(self, value):
        self._fontsize = value

    def box_width(self):
        width, height = self.drawer.max_layout_size(
            [i.name for i in self.qtile.groups],
            self.font,
            self.fontsize
        )
        return width + self.padding*2 + self.margin_x*2 + self.borderwidth*2

    def _configure(self, qtile, bar):
        base._Widget._configure(self, qtile, bar)
        self.layout = self.drawer.textlayout("", "ffffff", self.font, self.fontsize)
        self.setup_hooks()

    def setup_hooks(self):
        def hook_response(*args, **kwargs):
            self.bar.draw()
        hook.subscribe.client_managed(hook_response)
        hook.subscribe.client_urgent_hint_changed(hook_response)
        hook.subscribe.client_killed(hook_response)
        hook.subscribe.setgroup(hook_response)
        hook.subscribe.group_window_add(hook_response)

    def drawbox(self, offset, text, bordercolor, textcolor, width=None):
        self.layout.text = text
        self.layout.font_family = self.font
        self.layout.font_size = self.fontsize
        self.layout.colour = textcolor
        if width is not None:
            self.layout.width = width
        framed = self.layout.framed(self.borderwidth, bordercolor, self.padding, self.padding)
        framed.draw(offset, self.margin_y)

class AGroupBox(_GroupBase):
    """
        A widget that graphically displays the current group.
    """
    defaults = manager.Defaults(
        ("margin_y", 3, "Y margin outside the box"),
        ("margin_x", 3, "X margin outside the box"),
        ("borderwidth", 3, "Current group border width"),
        ("font", "Arial", "Font face"),
        ("fontsize", None, "Font pixel size - calculated if None"),
        ("foreground", "aaaaaa", "Font colour"),
        ("background", "000000", "Widget background"),
        ("border", "215578", "Border colour"),
        ("padding", 5, "Padding inside the box")
    )
    def click(self, x, y, button):
        self.bar.screen.group.cmd_nextgroup()

    def calculate_width(self):
        return self.box_width()

    def draw(self):
        self.drawer.clear(self.background)
        e = (i for i in self.qtile.groups if i.name == self.bar.screen.group.name ).next()
        self.drawbox(self.margin_x, e.name, self.border, self.foreground)
        self.drawer.draw(self.offset, self.width)

class GroupBox(_GroupBase):
    """
        A widget that graphically displays the current group.
    """
    defaults = manager.Defaults(
        ("active", "FFFFFF", "Active group font colour"),
        ("inactive", "404040", "Inactive group font colour"),
        ("margin_y", 3, "Y margin outside the box"),
        ("margin_x", 3, "X margin outside the box"),
        ("borderwidth", 3, "Current group border width"),
        ("font", "Arial", "Font face"),
        ("fontsize", None, "Font pixel size - calculated if None"),
        ("background", "000000", "Widget background"),
        ("this_screen_border", "215578", "Border colour for group on this screen."),
        ("other_screen_border", "404040", "Border colour for group on other screen."),
        ("padding", 5, "Padding inside the box"),
        ("urgent_border", "FF0000", "Urgent border color")
    )
    def __init__(self, **config):
        base._Widget.__init__(self, bar.CALCULATED, **config)

    def click(self, x, y, button):
        group = None
        curGroup = self.qtile.currentGroup
        if button == 5:
            group = curGroup.prevGroup()
        elif button == 4:
            group = curGroup.nextGroup()
        elif button == 1:
            groupOffset = int(x/self.box_width())
            if len(self.qtile.groups) - 1 >= groupOffset:
                group = self.qtile.groups[groupOffset]

        self.bar.screen.setGroup(group)

    def calculate_width(self):
        return self.box_width() * len(self.qtile.groups)

    def group_has_urgent(self, group):
        return len([w for w in group.windows if w.urgent]) > 0

    def draw(self):
        bw = self.box_width()
        self.drawer.clear(self.background)
        for i, g in enumerate(self.qtile.groups):
            if g.screen:
                if self.bar.screen.group.name == g.name:
                    border = self.this_screen_border
                else:
                    border = self.other_screen_border
            elif self.group_has_urgent(g):
                border = self.urgent_border
            else:
                border = self.background

            if g.windows:
                text = self.active
            else:
                text = self.inactive

            self.drawbox(
                self.margin_x + (bw*i),
                g.name,
                border,
                text,
                bw - self.margin_x*2 - self.padding*2
            )
        self.drawer.draw(self.offset, self.width)

<|endoftext|>


<commit_before>from datetime import datetime, timedelta
from dateutil import parser
from os.path import join
from tenyks.client import Client, run_client
from tenyks.client.config import settings
import sqlite3, requests

class TenyksWebpageMonitor(Client):

    irc_message_filters = {
        'add_url': [r'add url (.*)'],
        'list_url': r'list url',
        'del_url': r'delete url (.*)',
    }
    direct_only = True
    recurring_delay = 30

    def __init__(self, *args, **kwargs):
        super(TenyksWebpageMonitor, self).__init__(*args, **kwargs)
        self.create_tables(self.fetch_cursor())

    def fetch_cursor(self):
        db_file = 'eccentric_garbanzo.db'
        print "DB in {}".format(join(settings.WORKING_DIR, db_file))
        conn = sqlite3.connect(join(settings.WORKING_DIR, db_file))
        return conn.cursor()

    def handle(self, I, Dont, Care):
        pass

    def recurring(self):
        self.logger.debug('Checking pages')
        cur = self.fetch_cursor()
        for channel in self.get_channels(cur):
            connection = self.get_connection(cur, channel)
            for url in self.urls_by_channel(cur, channel):
                self.url_handler(cur, url, channel, connection)

    def handle_list_url(self, data, match):
        self.logger.debug('list_urls')
        cur = self.fetch_cursor()
        connection = self.get_or_create_connection(
                cur, data['connection'])
        channel = self.get_or_create_channel(
                cur, connection, data['target'])
        url_sql = """
            SELECT * FROM url
            WHERE channel_id = ?"""
        result = cur.execute(url_sql, (channel[0],)).fetchone()
        if not result:
            self.send('No urls monitored.', data)
        else:
            self.send('URLs for this channel:', data)
            for i, feed in enumerate(cur.execute(url_sql, (channel[0],))):
                self.send('{i}. {url}'.format(i=i+1,
                    url=feed[1]), data)

    def url_handler(self, cur, url, channel, connection):
        self.logger.debug('Checking: {}'.format(url[1]))
        should_send_alert = False
        error = None
        r = None

        try:
            r = requests.get(url[1])
        except requests.ConnectionError:
            error = " is unreachable!"

        if r and r.status_code in [404, 502, 503, 504, 401]:
            error = " is returning bad codes!"

        alert = self.get_most_recent_alert(cur, url[0])

        if alert:
            timestamp = parser.parse(alert[1])
            should_send_alert = datetime.utcnow() > timestamp + timedelta(minutes=10)

        if error is not None and should_send_alert:
            message = "\x034ALERT\x03: {url}{error} Code: {code}".format(
                url=url[1], error=error, code=getattr(r, "status_code", "N/A"))
            data = {
                'command': 'PRIVMSG',
                'client': self.name,
                'payload': message,
                'target': channel[1],
                'connection': connection[1],
            }
            if channel[1].startswith('#'):
                data['private_message'] = False
            else:
                data['private_message'] = True
                data['nick'] = channel[1]
            result = cur.execute("""
                INSERT INTO alert (url_id)
                VALUES (?)
            """, (url[0],))
            cur.connection.commit()
            self.send(message, data)

    def handle_add_url(self, data, match):
        if data['admin']:
            url = match.groups()[0]
            self.logger.debug('add_url: {}'.format(url))
            cur = self.fetch_cursor()
            connection = self.get_or_create_connection(cur,
                    data['connection'])
            if data['private_message']:
                target = data['nick']
            else:
                target = data['target']
            channel = self.get_or_create_channel(cur,
                    connection, target)
            url = self.get_or_create_url(cur, channel, url)
            self.send('{} is now monitored.'.format(url[1]), data)

    def handle_del_url(self, data, match):
        if data['admin']:
            url = match.groups()[0]
            self.logger.debug('del_url: {}'.format(url))
            cur = self.fetch_cursor()
            connection = self.get_or_create_connection(cur,
                data['connection'])
            channel = self.get_or_create_channel(cur,
                connection, data['target'])
            if self.url_exists(cur, url, channel):
                self.delete_url(cur, url, channel)
                self.send('{} is now deleted.'.format(url), data)
            else:
                self.send('{} did not exist.'.format(url), data)

    def url_exists(self, cur, url, channel):
        result = cur.execute("""
            SELECT * FROM url
            WHERE channel_id = ?
            AND url = ?
        """, (channel[0], url))
        return result.fetchone() is not None

    def delete_url(self, cur, url, channel):
        result = cur.execute("""
            DELETE FROM url
            WHERE channel_id = ?
            AND url = ?
        """, (channel[0], url))
        cur.connection.commit()

    def get_db(self):
        db_file = '{name}.db'.format(name=self.name)
        return sqlite3.connect(join(settings.WORKING_DIR, db_file))

    def get_most_recent_alert(self, cur, url):
        result = cur.execute("""
            SELECT * FROM alert WHERE url_id = ? ORDER BY created_at DESC;
        """, (url,))
        return result.fetchone()

    def get_channels(self, cur):
        result = cur.execute("""
            SELECT * FROM channel
        """)
        return result.fetchall()

    def get_connection(self, cur, channel):
        return cur.execute("""
            SELECT * FROM connection
            WHERE id = ?""", (channel[2],)).fetchone()

    def get_or_create_channel(self, cur, connection, channel_name):
        channel_sql = """
            SELECT * FROM channel
            WHERE channel = ?
            AND connection_id = ?"""
        result = cur.execute(channel_sql, (channel_name, connection[0]))
        channel = result.fetchone()
        if not channel:
            result = cur.execute("""
            INSERT INTO channel (channel, connection_id)
            VALUES (?, ?)""", (channel_name, connection[0]))
            result = cur.execute(channel_sql, (channel_name, connection[0]))
            cur.connection.commit()
            channel = result.fetchone()
        return channel

    def get_or_create_url(self, cur, channel, passed_url):
        url_sql = """
            SELECT * FROM url
            WHERE channel_id = ?
            AND url = ?
        """
        result = cur.execute(url_sql, (channel[0], passed_url))
        url = result.fetchone()
        if not url:
            result = cur.execute("""
            INSERT INTO url (channel_id, url)
            VALUES (?, ?)""", (channel[0], passed_url))
            cur.connection.commit()
            result = cur.execute(url_sql, (channel[0], passed_url))
            url = result.fetchone()
        return url

    def get_or_create_connection(self, cur, name):
        connection_sql = """
            SELECT *
            FROM connection
            WHERE connection_name = ?"""
        result = cur.execute(connection_sql, (name,))
        connection = result.fetchone()
        if not connection:
            result = cur.execute("""
                INSERT INTO connection (connection_name)
                VALUES (?)
            """, (name,))
            result = cur.execute(connection_sql, (name,))
            cur.connection.commit()
            connection = result.fetchone()
        return connection

    def urls_by_channel(self, cur, channel):
        result = cur.execute("""
            SELECT * FROM url
            WHERE channel_id = ?""", (channel[0],))
        return result.fetchall()

    def create_tables(self, db):
        table_sql = """
        CREATE TABLE IF NOT EXISTS alert (
            id INTEGER PRIMARY KEY,
            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
            url_id INTEGER,
            FOREIGN KEY(url_id)
                REFERENCES url(id)
        );
        CREATE TABLE IF NOT EXISTS url (
            id INTEGER PRIMARY KEY,
            url TEXT,
            channel_id INTEGER,
            FOREIGN KEY(channel_id)
                REFERENCES channel(id)
        );
        CREATE TABLE IF NOT EXISTS connection (
            id INTEGER PRIMARY KEY,
            connection_name TEXT
        );
        CREATE TABLE IF NOT EXISTS channel (
            id INTEGER PRIMARY KEY,
            channel TEXT,
            connection_id INTEGER,
            FOREIGN KEY(connection_id)
                REFERENCES connection(id)
        );
        """
        db.executescript(table_sql)

def main():
    run_client(TenyksWebpageMonitor)

if __name__ == '__main__':
    main()
<commit_msg>Added 500 to "Bad error codes" list<commit_after>from datetime import datetime, timedelta
from dateutil import parser
from os.path import join
from tenyks.client import Client, run_client
from tenyks.client.config import settings
import sqlite3, requests

class TenyksWebpageMonitor(Client):

    irc_message_filters = {
        'add_url': [r'add url (.*)'],
        'list_url': r'list url',
        'del_url': r'delete url (.*)',
    }
    direct_only = True
    recurring_delay = 30

    def __init__(self, *args, **kwargs):
        super(TenyksWebpageMonitor, self).__init__(*args, **kwargs)
        self.create_tables(self.fetch_cursor())

    def fetch_cursor(self):
        db_file = 'eccentric_garbanzo.db'
        print "DB in {}".format(join(settings.WORKING_DIR, db_file))
        conn = sqlite3.connect(join(settings.WORKING_DIR, db_file))
        return conn.cursor()

    def handle(self, I, Dont, Care):
        pass

    def recurring(self):
        self.logger.debug('Checking pages')
        cur = self.fetch_cursor()
        for channel in self.get_channels(cur):
            connection = self.get_connection(cur, channel)
            for url in self.urls_by_channel(cur, channel):
                self.url_handler(cur, url, channel, connection)

    def handle_list_url(self, data, match):
        self.logger.debug('list_urls')
        cur = self.fetch_cursor()
        connection = self.get_or_create_connection(
                cur, data['connection'])
        channel = self.get_or_create_channel(
                cur, connection, data['target'])
        url_sql = """
            SELECT * FROM url
            WHERE channel_id = ?"""
        result = cur.execute(url_sql, (channel[0],)).fetchone()
        if not result:
            self.send('No urls monitored.', data)
        else:
            self.send('URLs for this channel:', data)
            for i, feed in enumerate(cur.execute(url_sql, (channel[0],))):
                self.send('{i}. {url}'.format(i=i+1,
                    url=feed[1]), data)

    def url_handler(self, cur, url, channel, connection):
        self.logger.debug('Checking: {}'.format(url[1]))
        should_send_alert = False
        error = None
        r = None

        try:
            r = requests.get(url[1])
        except requests.ConnectionError:
            error = " is unreachable!"

        if r and r.status_code in [500, 404, 502, 503, 504, 401]:
            error = " is returning bad codes!"

        alert = self.get_most_recent_alert(cur, url[0])

        if alert:
            timestamp = parser.parse(alert[1])
            should_send_alert = datetime.utcnow() > timestamp + timedelta(minutes=10)

        if error is not None and should_send_alert:
            message = "\x034ALERT\x03: {url}{error} Code: {code}".format(
                url=url[1], error=error, code=getattr(r, "status_code", "N/A"))
            data = {
                'command': 'PRIVMSG',
                'client': self.name,
                'payload': message,
                'target': channel[1],
                'connection': connection[1],
            }
            if channel[1].startswith('#'):
                data['private_message'] = False
            else:
                data['private_message'] = True
                data['nick'] = channel[1]
            result = cur.execute("""
                INSERT INTO alert (url_id)
                VALUES (?)
            """, (url[0],))
            cur.connection.commit()
            self.send(message, data)

    def handle_add_url(self, data, match):
        if data['admin']:
            url = match.groups()[0]
            self.logger.debug('add_url: {}'.format(url))
            cur = self.fetch_cursor()
            connection = self.get_or_create_connection(cur,
                    data['connection'])
            if data['private_message']:
                target = data['nick']
            else:
                target = data['target']
            channel = self.get_or_create_channel(cur,
                    connection, target)
            url = self.get_or_create_url(cur, channel, url)
            self.send('{} is now monitored.'.format(url[1]), data)

    def handle_del_url(self, data, match):
        if data['admin']:
            url = match.groups()[0]
            self.logger.debug('del_url: {}'.format(url))
            cur = self.fetch_cursor()
            connection = self.get_or_create_connection(cur,
                data['connection'])
            channel = self.get_or_create_channel(cur,
                connection, data['target'])
            if self.url_exists(cur, url, channel):
                self.delete_url(cur, url, channel)
                self.send('{} is now deleted.'.format(url), data)
            else:
                self.send('{} did not exist.'.format(url), data)

    def url_exists(self, cur, url, channel):
        result = cur.execute("""
            SELECT * FROM url
            WHERE channel_id = ?
            AND url = ?
        """, (channel[0], url))
        return result.fetchone() is not None

    def delete_url(self, cur, url, channel):
        result = cur.execute("""
            DELETE FROM url
            WHERE channel_id = ?
            AND url = ?
        """, (channel[0], url))
        cur.connection.commit()

    def get_db(self):
        db_file = '{name}.db'.format(name=self.name)
        return sqlite3.connect(join(settings.WORKING_DIR, db_file))

    def get_most_recent_alert(self, cur, url):
        result = cur.execute("""
            SELECT * FROM alert WHERE url_id = ? ORDER BY created_at DESC;
        """, (url,))
        return result.fetchone()

    def get_channels(self, cur):
        result = cur.execute("""
            SELECT * FROM channel
        """)
        return result.fetchall()

    def get_connection(self, cur, channel):
        return cur.execute("""
            SELECT * FROM connection
            WHERE id = ?""", (channel[2],)).fetchone()

    def get_or_create_channel(self, cur, connection, channel_name):
        channel_sql = """
            SELECT * FROM channel
            WHERE channel = ?
            AND connection_id = ?"""
        result = cur.execute(channel_sql, (channel_name, connection[0]))
        channel = result.fetchone()
        if not channel:
            result = cur.execute("""
            INSERT INTO channel (channel, connection_id)
            VALUES (?, ?)""", (channel_name, connection[0]))
            result = cur.execute(channel_sql, (channel_name, connection[0]))
            cur.connection.commit()
            channel = result.fetchone()
        return channel

    def get_or_create_url(self, cur, channel, passed_url):
        url_sql = """
            SELECT * FROM url
            WHERE channel_id = ?
            AND url = ?
        """
        result = cur.execute(url_sql, (channel[0], passed_url))
        url = result.fetchone()
        if not url:
            result = cur.execute("""
            INSERT INTO url (channel_id, url)
            VALUES (?, ?)""", (channel[0], passed_url))
            cur.connection.commit()
            result = cur.execute(url_sql, (channel[0], passed_url))
            url = result.fetchone()
        return url

    def get_or_create_connection(self, cur, name):
        connection_sql = """
            SELECT *
            FROM connection
            WHERE connection_name = ?"""
        result = cur.execute(connection_sql, (name,))
        connection = result.fetchone()
        if not connection:
            result = cur.execute("""
                INSERT INTO connection (connection_name)
                VALUES (?)
            """, (name,))
            result = cur.execute(connection_sql, (name,))
            cur.connection.commit()
            connection = result.fetchone()
        return connection

    def urls_by_channel(self, cur, channel):
        result = cur.execute("""
            SELECT * FROM url
            WHERE channel_id = ?""", (channel[0],))
        return result.fetchall()

    def create_tables(self, db):
        table_sql = """
        CREATE TABLE IF NOT EXISTS alert (
            id INTEGER PRIMARY KEY,
            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
            url_id INTEGER,
            FOREIGN KEY(url_id)
                REFERENCES url(id)
        );
        CREATE TABLE IF NOT EXISTS url (
            id INTEGER PRIMARY KEY,
            url TEXT,
            channel_id INTEGER,
            FOREIGN KEY(channel_id)
                REFERENCES channel(id)
        );
        CREATE TABLE IF NOT EXISTS connection (
            id INTEGER PRIMARY KEY,
            connection_name TEXT
        );
        CREATE TABLE IF NOT EXISTS channel (
            id INTEGER PRIMARY KEY,
            channel TEXT,
            connection_id INTEGER,
            FOREIGN KEY(connection_id)
                REFERENCES connection(id)
        );
        """
        db.executescript(table_sql)

def main():
    run_client(TenyksWebpageMonitor)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import re
from collections import OrderedDict

from bs4 import BeautifulSoup

from wikicurses import formats
from wikicurses import settings

skipclass = ('wiki-sidebar', 'infobox', 'mw-editsection', 'editsection',
             'wikitable', 'thumb', 'gallery', 'article-thumb',
             'mw-headline-anchor')
skipsection = ('Contents', 'External links', 'See also')

if settings.hide_references:
    skipclass += ('reference',)
    skipsection += ('References',)

class UrwidMarkupHandler:

    def __init__(self):
        self._list = []
        self._oldlist = None

    def add(self, text, attribute):
        if not text:
            return
        if self and self[-1][0] == attribute:
            self[-1][1] += text
        else:
            self._list.append([attribute, text])

    def search(self, findtext):
        self.unsearch()
        self._oldlist = self._list
        self._list = []
        for attribute, text in self._oldlist:
            cur = 0
            for match in re.finditer(findtext, text):
                start, end = match.start(), match.end()
                self.add(text[cur:start], attribute)
                self.add(text[start:end], attribute | formats.searchresult)
                cur = end
            if text[cur:]:
                self.add(text[cur:], attribute)

    def unsearch(self):
        if self._oldlist:
            self._list = self._oldlist
            self._oldlist = None

    def __iter__(self):
        return (tuple(i) for i in self._list)

    def __len__(self):
        return len(self._list)

    def __getitem__(self, key):
        return self._list[key]

def _processArticleSection(section):
    items = UrwidMarkupHandler()
    for i in section:
        if isinstance(i, str):
            strings = (i,)
        else:
            if i.name == 'h2' or i.find('h2'):
                break
            strings = i.strings
        for item in strings:
            partags = {i.name for i in item.parents}
            tformat = sum(
                formats[i] for i in set(i.name for i in formats).intersection(partags))
            if [i for i in partags if re.fullmatch('h[3-6]', i)]:
                tformat |= formats.h
            items.add(item, tformat)
    if items:
        items[0][1] = items[0][1].lstrip()
        items[-1][1] = items[-1][1].rstrip() + '\n\n'
    return items

def parseArticle(html):
    """Parse article html and return OrderedDict of sections."""
    html = html.replace('\t', ' ')
    sections = OrderedDict()
    soup = BeautifulSoup(html, 'lxml')
    # Turn into tuple since the iterator is being modified
    for i in tuple(soup.strings):
        if not {'pre', 'code'}.intersection(j.name for j in i.parents):
            i.replace_with(i.replace('\n', ''))
    for i in soup.find_all('p'):
        i.insert_after(soup.new_string('\n\n'))
    for i in soup.find_all(['h2', 'h3', 'h4', 'h5', 'h6', 'br']):
        i.insert_after(soup.new_string('\n'))
    for i in soup.find_all('div'):
        if i.text and not i.find_all('div'):
            i.insert_after(soup.new_string('\n'))
    for i in soup.find_all('li'):
        if i.parent.name == 'ol':
            num = i.parent.find_all('li').index(i) + 1
            i.insert_before(soup.new_string(str(num) + '. '))
        else:
            i.insert_before(soup.new_string('• '))
        i.insert_after(soup.new_string('\n'))
        if 'li' not in (j.name for j in i.next_siblings): # Last item in list
            i.insert_after(soup.new_string('\n'))
    for i in soup.find_all(True, class_=skipclass):
        i.decompose()
    sections[''] = _processArticleSection(soup.body or soup)
    for i in soup.find_all('h2'):
        if i.text not in skipsection:
            sections[i.text.strip()] = _processArticleSection(i.next_siblings)
    for i in sections:
        if not sections[i]:
            del sections[i]
    return sections

def parseFeature(html):
    """Parse featured feed html by striping out html tags."""
    items = UrwidMarkupHandler()
    text = BeautifulSoup(html, 'lxml').text.strip() + '\n\n'
    text = re.sub('\n\n+', '\n\n', text)
    items.add(text, 0)
    return items

def _processDisambigSection(section):
    items = []
    for i in section:
        if isinstance(i, str):
            continue
        if i.name == 'h2' or i.find('h2'):
            break
        for item in i.find_all('li'):
            items.append(
                (item.a.text if item.a else '', item.text.split('\n')[0]))
    return items

def parseDisambig(html):
    """Parse disambiguation page and return list of (article, text) tuples."""
    sections = OrderedDict()
    soup = BeautifulSoup(html, 'lxml')
    for i in soup.find_all(True, class_=skipclass):
        i.decompose()
    sections[''] = _processDisambigSection(soup)
    for i in soup.find_all('h2'):
        if i.text not in ('Contents', 'See also'):
            sections[i.text] = _processDisambigSection(i.next_siblings)
    return sections
<commit_msg>Strip out html comments<commit_after>import re
from collections import OrderedDict

import bs4

from wikicurses import formats
from wikicurses import settings

skipclass = ('wiki-sidebar', 'infobox', 'mw-editsection', 'editsection',
             'wikitable', 'thumb', 'gallery', 'article-thumb',
             'mw-headline-anchor')
skipsection = ('Contents', 'External links', 'See also')

if settings.hide_references:
    skipclass += ('reference',)
    skipsection += ('References',)

class UrwidMarkupHandler:

    def __init__(self):
        self._list = []
        self._oldlist = None

    def add(self, text, attribute):
        if not text:
            return
        if self and self[-1][0] == attribute:
            self[-1][1] += text
        else:
            self._list.append([attribute, text])

    def search(self, findtext):
        self.unsearch()
        self._oldlist = self._list
        self._list = []
        for attribute, text in self._oldlist:
            cur = 0
            for match in re.finditer(findtext, text):
                start, end = match.start(), match.end()
                self.add(text[cur:start], attribute)
                self.add(text[start:end], attribute | formats.searchresult)
                cur = end
            if text[cur:]:
                self.add(text[cur:], attribute)

    def unsearch(self):
        if self._oldlist:
            self._list = self._oldlist
            self._oldlist = None

    def __iter__(self):
        return (tuple(i) for i in self._list)

    def __len__(self):
        return len(self._list)

    def __getitem__(self, key):
        return self._list[key]

def _processArticleSection(section):
    items = UrwidMarkupHandler()
    for i in section:
        if isinstance(i, str):
            strings = (i,)
        else:
            if i.name == 'h2' or i.find('h2'):
                break
            strings = i.strings
        for item in strings:
            if isinstance(item, bs4.element.Comment):
                continue # Strip out html comments
            partags = {i.name for i in item.parents}
            tformat = sum(
                formats[i] for i in set(i.name for i in formats).intersection(partags))
            if [i for i in partags if re.fullmatch('h[3-6]', i)]:
                tformat |= formats.h
            items.add(item, tformat)
    if items:
        items[0][1] = items[0][1].lstrip()
        items[-1][1] = items[-1][1].rstrip() + '\n\n'
    return items

def parseArticle(html):
    """Parse article html and return OrderedDict of sections."""
    html = html.replace('\t', ' ')
    sections = OrderedDict()
    soup = bs4.BeautifulSoup(html, 'lxml')
    # Turn into tuple since the iterator is being modified
    for i in tuple(soup.strings):
        if not {'pre', 'code'}.intersection(j.name for j in i.parents):
            i.replace_with(i.replace('\n', ''))
    for i in soup.find_all('p'):
        i.insert_after(soup.new_string('\n\n'))
    for i in soup.find_all(['h2', 'h3', 'h4', 'h5', 'h6', 'br']):
        i.insert_after(soup.new_string('\n'))
    for i in soup.find_all('div'):
        if i.text and not i.find_all('div'):
            i.insert_after(soup.new_string('\n'))
    for i in soup.find_all('li'):
        if i.parent.name == 'ol':
            num = i.parent.find_all('li').index(i) + 1
            i.insert_before(soup.new_string(str(num) + '. '))
        else:
            i.insert_before(soup.new_string('• '))
        i.insert_after(soup.new_string('\n'))
        if 'li' not in (j.name for j in i.next_siblings): # Last item in list
            i.insert_after(soup.new_string('\n'))
    for i in soup.find_all(True, class_=skipclass):
        i.decompose()
    sections[''] = _processArticleSection(soup.body or soup)
    for i in soup.find_all('h2'):
        if i.text not in skipsection:
            sections[i.text.strip()] = _processArticleSection(i.next_siblings)
    for i in sections:
        if not sections[i]:
            del sections[i]
    return sections

def parseFeature(html):
    """Parse featured feed html by striping out html tags."""
    items = UrwidMarkupHandler()
    text = bs4.BeautifulSoup(html, 'lxml').text.strip() + '\n\n'
    text = re.sub('\n\n+', '\n\n', text)
    items.add(text, 0)
    return items

def _processDisambigSection(section):
    items = []
    for i in section:
        if isinstance(i, str):
            continue
        if i.name == 'h2' or i.find('h2'):
            break
        for item in i.find_all('li'):
            items.append(
                (item.a.text if item.a else '', item.text.split('\n')[0]))
    return items

def parseDisambig(html):
    """Parse disambiguation page and return list of (article, text) tuples."""
    sections = OrderedDict()
    soup = bs4.BeautifulSoup(html, 'lxml')
    for i in soup.find_all(True, class_=skipclass):
        i.decompose()
    sections[''] = _processDisambigSection(soup)
    for i in soup.find_all('h2'):
        if i.text not in ('Contents', 'See also'):
            sections[i.text] = _processDisambigSection(i.next_siblings)
    return sections
<|endoftext|>


<commit_before>import json
from functools import reduce
from collections import defaultdict
from node import Node, Interface
from link import Link, LinkConnector

class NodeDB:
  def __init__(self):
    self._nodes = []
    self._links = []

  # fetch list of links
  def get_links(self):
    self.update_vpn_links()
    return self.reduce_links()

  # fetch list of nodes
  def get_nodes(self):
    return self._nodes

  def maybe_node_by_fuzzy_mac(self, mac):
    mac_a = mac.lower()

    for node in self._nodes:
      for mac_b in node.macs:
        if is_derived_mac(mac_a, mac_b):
          return node

    raise KeyError

  def maybe_node_by_mac(self, macs):
    for node in self._nodes:
      for mac in macs:
        if mac.lower() in node.macs:
          return node

    raise KeyError

  def maybe_node_by_id(self, mac):
    for node in self._nodes:
      if mac.lower() == node.id:
        return node

    raise KeyError

  def parse_vis_data(self,vis_data):
    for x in vis_data:

      if 'of' in x:
        try:
          node = self.maybe_node_by_mac((x['of'], x['secondary']))
        except:
          node = Node()
          node.flags['online'] = True
          if 'legacy' in x:
            node.flags['legacy'] = True
          self._nodes.append(node)

        node.add_mac(x['of'])
        node.add_mac(x['secondary'])

    for x in vis_data:

      if 'router' in x:
        try:
          node = self.maybe_node_by_mac((x['router'], ))
        except:
          node = Node()
          node.flags['online'] = True
          if 'legacy' in x:
            node.flags['legacy'] = True
          node.add_mac(x['router'])
          self._nodes.append(node)

        # If it's a TT link and the MAC is very similar
        # consider this MAC as one of the routers
        # MACs
        if 'gateway' in x and x['label'] == "TT":
          if is_similar(x['router'], x['gateway']):
            node.add_mac(x['gateway'])

            # skip processing as regular link
            continue

        try:
          if 'neighbor' in x:
            try:
              node = self.maybe_node_by_mac((x['neighbor']))
            except:
              continue

          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          node = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          node = Node()
          node.flags['online'] = True
          if x['label'] == 'TT':
            node.flags['client'] = True

          node.add_mac(x['neighbor'])
          self._nodes.append(node)

    for x in vis_data:

      if 'router' in x:
        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          router = self.maybe_node_by_mac((x['router'], ))
          neighbor = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          continue

        # filter TT links merged in previous step
        if router == neighbor:
          continue

        link = Link()
        link.source = LinkConnector()
        link.source.interface = x['router']
        link.source.id = self._nodes.index(router)
        link.target = LinkConnector()
        link.target.interface = x['neighbor']
        link.target.id = self._nodes.index(neighbor)
        link.quality = x['label']
        link.id = "-".join(sorted((link.source.interface, link.target.interface)))

        if x['label'] == "TT":
          link.type = "client"

        self._links.append(link)

    for x in vis_data:

      if 'primary' in x:
        try:
          node = self.maybe_node_by_mac((x['primary'], ))
        except:
          continue

        node.id = x['primary']

  def reduce_links(self):
    tmp_links = defaultdict(list)

    for link in self._links:
      tmp_links[link.id].append(link)

    links = []

    def reduce_link(a, b):
      a.id = b.id
      a.source = b.source
      a.target = b.target
      a.type = b.type
      a.quality = ", ".join([x for x in (a.quality, b.quality) if x])

      return a

    for k, v in tmp_links.items():
      new_link = reduce(reduce_link, v, Link())
      links.append(new_link)

    return links

  def import_aliases(self, aliases):
    for mac, alias in aliases.items():
      try:
        node = self.maybe_node_by_fuzzy_mac(mac)
      except:
        # create an offline node
        node = Node()
        node.add_mac(mac)
        self._nodes.append(node)

      if 'name' in alias:
        node.name = alias['name']

      if 'vpn' in alias and alias['vpn']:
        node.interfaces[mac].vpn = True

      if 'gps' in alias:
        node.gps = alias['gps']

      if 'firmware' in alias:
        node.firmware = alias['firmware']

  # list of macs
  # if options['gateway']:
  #   mark_gateways(options['gateway'])
  def mark_gateways(self, gateways):
    for gateway in gateways:
      try:
        node = self.maybe_node_by_mac((gateway, ))
      except:
        continue

      node.flags['gateway'] = True

  def update_vpn_links(self):
    changes = 1
    while changes > 0:
      changes = 0
      for link in self._links:
        if link.type == "client":
          continue

        source_interface = self._nodes[link.source.id].interfaces[link.source.interface]
        target_interface = self._nodes[link.target.id].interfaces[link.target.interface]
        if source_interface.vpn or target_interface.vpn:
          source_interface.vpn = True
          target_interface.vpn = True
          if link.type != "vpn":
            changes += 1

          link.type = "vpn"

  def count_clients(self):
    for link in self._links:
      try:
        a = self.maybe_node_by_id(link.source.interface)
        b = self.maybe_node_by_id(link.target.interface)

        if a.flags['client']:
          client = a
          node = b
        elif b.flags['client']:
          client = b
          node = a
        else:
          continue

        node.clientcount += 1
      except:
        pass

  def obscure_clients(self):

    globalIdCounter = 0
    nodeCounters = {}
    clientIds = {}

    for node in self._nodes:
      if node.flags['client']:
        node.macs = set()
        clientIds[node.id] = None

    for link in self._links:
      ids = link.source.interface
      idt = link.target.interface

      try:
        node_source = self.maybe_node_by_fuzzy_mac(ids)
        node_target = self.maybe_node_by_id(idt)

        if not node_source.flags['client'] and not node_target.flags['client']:
          # if none of the nodes associated with this link are clients,
          # we do not want to obscure
          continue

        if ids in clientIds and idt in clientIds:
          # This is for corner cases, when a client
          # is linked to another client.
          clientIds[ids] = str(globalIdCounter)
          ids = str(globalIdCounter)
          globalIdCounter += 1

          clientIds[idt] = str(globalIdCounter)
          idt = str(globalIdCounter)
          globalIdCounter += 1

        elif ids in clientIds:
          newId = generateId(idt)
          clientIds[ids] = newId
          ids = newId

          link.source.interface = ids;
          node_source.id = ids;

        elif idt in clientIds:
          newId = generateId(ids,nodeCounters)
          clientIds[idt] = newId
          idt = newId

          link.target.interface = idt;
          node_target.id = idt;

        link.id = ids + "-" + idt

      except KeyError:
        pass

# extends node id by incremented node counter
def generateId(nodeId,nodeCounters):
  if nodeId in nodeCounters:
    n = nodeCounters[nodeId]
    nodeCounters[nodeId] = n + 1
  else:
    nodeCounters[nodeId] = 1
    n = 0

  return nodeId + "_" + str(n)

# compares two MACs and decides whether they are
# similar and could be from the same node
def is_similar(a, b):
  if a == b:
    return True

  try:
    mac_a = list(int(i, 16) for i in a.split(":"))
    mac_b = list(int(i, 16) for i in b.split(":"))
  except ValueError:
    return False

  # first byte must only differ in bit 2
  if mac_a[0] | 2 == mac_b[0] | 2:
    # count different bytes
    c = [x for x in zip(mac_a[1:], mac_b[1:]) if x[0] != x[1]]
  else:
    return False

  # no more than two additional bytes must differ
  if len(c) <= 2:
    delta = 0

  if len(c) > 0:
    delta = sum(abs(i[0] -i[1]) for i in c)

  # These addresses look pretty similar!
  return delta < 8

def is_derived_mac(a, b):
  if a == b:
    return True

  try:
    mac_a = list(int(i, 16) for i in a.split(":"))
    mac_b = list(int(i, 16) for i in b.split(":"))
  except ValueError:
    return False

  if mac_a[4] != mac_b[4] or mac_a[2] != mac_b[2] or mac_a[1] != mac_b[1]:
    return False

  x = list(mac_a)
  x[5] += 1
  x[5] %= 255
  if mac_b == x:
    return True

  x[0] |= 2
  if mac_b == x:
    return True

  x[3] += 1
  x[3] %= 255
  if mac_b == x:
    return True

  x = list(mac_a)
  x[0] |= 2
  x[5] += 2
  x[5] %= 255
  if mac_b == x:
    return True

  x = list(mac_a)
  x[0] |= 2
  x[3] += 1
  x[3] %= 255
  if mac_b == x:
    return True

  return False
<commit_msg>try non-fuzzy prior fuzzy mac matching<commit_after>import json
from functools import reduce
from collections import defaultdict
from node import Node, Interface
from link import Link, LinkConnector

class NodeDB:
  def __init__(self):
    self._nodes = []
    self._links = []

  # fetch list of links
  def get_links(self):
    self.update_vpn_links()
    return self.reduce_links()

  # fetch list of nodes
  def get_nodes(self):
    return self._nodes

  def maybe_node_by_fuzzy_mac(self, mac):
    mac_a = mac.lower()

    for node in self._nodes:
      for mac_b in node.macs:
        if is_derived_mac(mac_a, mac_b):
          return node

    raise KeyError

  def maybe_node_by_mac(self, macs):
    for node in self._nodes:
      for mac in macs:
        if mac.lower() in node.macs:
          return node

    raise KeyError

  def maybe_node_by_id(self, mac):
    for node in self._nodes:
      if mac.lower() == node.id:
        return node

    raise KeyError

  def parse_vis_data(self,vis_data):
    for x in vis_data:

      if 'of' in x:
        try:
          node = self.maybe_node_by_mac((x['of'], x['secondary']))
        except:
          node = Node()
          node.flags['online'] = True
          if 'legacy' in x:
            node.flags['legacy'] = True
          self._nodes.append(node)

        node.add_mac(x['of'])
        node.add_mac(x['secondary'])

    for x in vis_data:

      if 'router' in x:
        try:
          node = self.maybe_node_by_mac((x['router'], ))
        except:
          node = Node()
          node.flags['online'] = True
          if 'legacy' in x:
            node.flags['legacy'] = True
          node.add_mac(x['router'])
          self._nodes.append(node)

        # If it's a TT link and the MAC is very similar
        # consider this MAC as one of the routers
        # MACs
        if 'gateway' in x and x['label'] == "TT":
          if is_similar(x['router'], x['gateway']):
            node.add_mac(x['gateway'])

            # skip processing as regular link
            continue

        try:
          if 'neighbor' in x:
            try:
              node = self.maybe_node_by_mac((x['neighbor']))
            except:
              continue

          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          node = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          node = Node()
          node.flags['online'] = True
          if x['label'] == 'TT':
            node.flags['client'] = True

          node.add_mac(x['neighbor'])
          self._nodes.append(node)

    for x in vis_data:

      if 'router' in x:
        try:
          if 'gateway' in x:
            x['neighbor'] = x['gateway']

          router = self.maybe_node_by_mac((x['router'], ))
          neighbor = self.maybe_node_by_mac((x['neighbor'], ))
        except:
          continue

        # filter TT links merged in previous step
        if router == neighbor:
          continue

        link = Link()
        link.source = LinkConnector()
        link.source.interface = x['router']
        link.source.id = self._nodes.index(router)
        link.target = LinkConnector()
        link.target.interface = x['neighbor']
        link.target.id = self._nodes.index(neighbor)
        link.quality = x['label']
        link.id = "-".join(sorted((link.source.interface, link.target.interface)))

        if x['label'] == "TT":
          link.type = "client"

        self._links.append(link)

    for x in vis_data:

      if 'primary' in x:
        try:
          node = self.maybe_node_by_mac((x['primary'], ))
        except:
          continue

        node.id = x['primary']

  def reduce_links(self):
    tmp_links = defaultdict(list)

    for link in self._links:
      tmp_links[link.id].append(link)

    links = []

    def reduce_link(a, b):
      a.id = b.id
      a.source = b.source
      a.target = b.target
      a.type = b.type
      a.quality = ", ".join([x for x in (a.quality, b.quality) if x])

      return a

    for k, v in tmp_links.items():
      new_link = reduce(reduce_link, v, Link())
      links.append(new_link)

    return links

  def import_aliases(self, aliases):
    for mac, alias in aliases.items():
      try:
        node = self.maybe_node_by_mac([mac])
      except:
        try:
          node = self.maybe_node_by_fuzzy_mac(mac)
        except:
          # create an offline node
          node = Node()
          node.add_mac(mac)
          self._nodes.append(node)

      if 'name' in alias:
        node.name = alias['name']

      if 'vpn' in alias and alias['vpn']:
        node.interfaces[mac].vpn = True

      if 'gps' in alias:
        node.gps = alias['gps']

      if 'firmware' in alias:
        node.firmware = alias['firmware']

  # list of macs
  # if options['gateway']:
  #   mark_gateways(options['gateway'])
  def mark_gateways(self, gateways):
    for gateway in gateways:
      try:
        node = self.maybe_node_by_mac((gateway, ))
      except:
        continue

      node.flags['gateway'] = True

  def update_vpn_links(self):
    changes = 1
    while changes > 0:
      changes = 0
      for link in self._links:
        if link.type == "client":
          continue

        source_interface = self._nodes[link.source.id].interfaces[link.source.interface]
        target_interface = self._nodes[link.target.id].interfaces[link.target.interface]
        if source_interface.vpn or target_interface.vpn:
          source_interface.vpn = True
          target_interface.vpn = True
          if link.type != "vpn":
            changes += 1

          link.type = "vpn"

  def count_clients(self):
    for link in self._links:
      try:
        a = self.maybe_node_by_id(link.source.interface)
        b = self.maybe_node_by_id(link.target.interface)

        if a.flags['client']:
          client = a
          node = b
        elif b.flags['client']:
          client = b
          node = a
        else:
          continue

        node.clientcount += 1
      except:
        pass

  def obscure_clients(self):

    globalIdCounter = 0
    nodeCounters = {}
    clientIds = {}

    for node in self._nodes:
      if node.flags['client']:
        node.macs = set()
        clientIds[node.id] = None

    for link in self._links:
      ids = link.source.interface
      idt = link.target.interface

      try:
        node_source = self.maybe_node_by_fuzzy_mac(ids)
        node_target = self.maybe_node_by_id(idt)

        if not node_source.flags['client'] and not node_target.flags['client']:
          # if none of the nodes associated with this link are clients,
          # we do not want to obscure
          continue

        if ids in clientIds and idt in clientIds:
          # This is for corner cases, when a client
          # is linked to another client.
          clientIds[ids] = str(globalIdCounter)
          ids = str(globalIdCounter)
          globalIdCounter += 1

          clientIds[idt] = str(globalIdCounter)
          idt = str(globalIdCounter)
          globalIdCounter += 1

        elif ids in clientIds:
          newId = generateId(idt)
          clientIds[ids] = newId
          ids = newId

          link.source.interface = ids;
          node_source.id = ids;

        elif idt in clientIds:
          newId = generateId(ids,nodeCounters)
          clientIds[idt] = newId
          idt = newId

          link.target.interface = idt;
          node_target.id = idt;

        link.id = ids + "-" + idt

      except KeyError:
        pass

# extends node id by incremented node counter
def generateId(nodeId,nodeCounters):
  if nodeId in nodeCounters:
    n = nodeCounters[nodeId]
    nodeCounters[nodeId] = n + 1
  else:
    nodeCounters[nodeId] = 1
    n = 0

  return nodeId + "_" + str(n)

# compares two MACs and decides whether they are
# similar and could be from the same node
def is_similar(a, b):
  if a == b:
    return True

  try:
    mac_a = list(int(i, 16) for i in a.split(":"))
    mac_b = list(int(i, 16) for i in b.split(":"))
  except ValueError:
    return False

  # first byte must only differ in bit 2
  if mac_a[0] | 2 == mac_b[0] | 2:
    # count different bytes
    c = [x for x in zip(mac_a[1:], mac_b[1:]) if x[0] != x[1]]
  else:
    return False

  # no more than two additional bytes must differ
  if len(c) <= 2:
    delta = 0

  if len(c) > 0:
    delta = sum(abs(i[0] -i[1]) for i in c)

  # These addresses look pretty similar!
  return delta < 8

def is_derived_mac(a, b):
  if a == b:
    return True

  try:
    mac_a = list(int(i, 16) for i in a.split(":"))
    mac_b = list(int(i, 16) for i in b.split(":"))
  except ValueError:
    return False

  if mac_a[4] != mac_b[4] or mac_a[2] != mac_b[2] or mac_a[1] != mac_b[1]:
    return False

  x = list(mac_a)
  x[5] += 1
  x[5] %= 255
  if mac_b == x:
    return True

  x[0] |= 2
  if mac_b == x:
    return True

  x[3] += 1
  x[3] %= 255
  if mac_b == x:
    return True

  x = list(mac_a)
  x[0] |= 2
  x[5] += 2
  x[5] %= 255
  if mac_b == x:
    return True

  x = list(mac_a)
  x[0] |= 2
  x[3] += 1
  x[3] %= 255
  if mac_b == x:
    return True

  return False
<|endoftext|>