<commit_before>import kivy
import sqlite3
import string
from kivy.app import App
from kivy.uix.widget import Widget
from kivy.uix.label import Label
from kivy.uix.button import Button
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.gridlayout import GridLayout
from kivy.properties import ObjectProperty
from kivy.properties import StringProperty
from kivy.uix.scrollview import ScrollView
from kivy.core.window import Window

# if there is a need to load another kv file
#from kivy.lang import Builder
#Builder.load_file('MB2.kv')

sqlite_db = "C:/Users/USER-MB/source/GIT_Repo for myKivyApp01/myKivyApp01/myKivyAppDB.db"
conn = sqlite3.connect(sqlite_db)
c = conn.cursor()

class SubControlButtons(Button):
    theRoot = ObjectProperty(None)

class FamilyButtons(Button):
    theRoot = ObjectProperty(None)

class RootContainer(BoxLayout):
    instance1 = ObjectProperty(None)
    instance2 = ObjectProperty(None)

    def __init__(self, **kwargs):
        super(RootContainer, self).__init__(**kwargs)
        #create buttons from query result
        self.lbl5.add_widget(Button(text="ACC"))

    def clickAction2(self, instance2):
        self.lbl2.text = instance2.text
        #pass

    def clickAction1(self, instance1):
        #print(instance1)
        #identify the button pressed
        t = (1,)
        buttonText = (instance1.text[0:2],)
        print(buttonText)

        #use button text to query all related button info from DB
        c.execute('select * from moderateBaseline where family=?',buttonText)

        #fetch results of query, fetchall() can only be used once
        results = c.fetchall()
        #print(results)
        #print(results[0][3])
        self.lbl2.text = results[1][4]
        #bind root so we can identify on_press in the buttons below
        #self.bind(self)
        # set height of grid before populating with buttons
        self.lbl3.bind(minimum_height=self.lbl3.setter('height'))
        print("lbl3.bind complete")
        # clear all old children
        self.lbl3.clear_widgets()
        print("lbl3 cleared")
        # dynamically add buttons
        for x in results:
            #self.lbl3.add_widget(Button(text=str(x[3])))
            self.lbl3.add_widget(SubControlButtons(text=str(x[3]), theRoot = self))

class MBApp(App):
    # this is a native function from Kivy to actually build the app using KV files
    def build(self):
        #call RootContainer
        return RootContainer()

if __name__ == '__main__':
    MBApp().run()<commit_msg>20171126<commit_after>import kivy
import sqlite3
import string
from kivy.app import App
from kivy.uix.widget import Widget
from kivy.uix.label import Label
from kivy.uix.button import Button
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.gridlayout import GridLayout
from kivy.properties import ObjectProperty
from kivy.properties import StringProperty
from kivy.uix.scrollview import ScrollView
from kivy.core.window import Window

# if there is a need to load another kv file
#from kivy.lang import Builder
#Builder.load_file('MB2.kv')

sqlite_db = "C:/Users/USER-MB/source/GIT_Repo for myKivyApp01/myKivyApp01/myKivyAppDB.db"
conn = sqlite3.connect(sqlite_db)
c = conn.cursor()

class SubControlButtons(Button):
    theRoot = ObjectProperty(None)

class FamilyButtons(Button):
    theRoot = ObjectProperty(None)

class RootContainer(BoxLayout):
    instance1 = ObjectProperty(None)
    instance2 = ObjectProperty(None)

    def __init__(self, **kwargs):
        super(RootContainer, self).__init__(**kwargs)
        #create buttons from query result
        self.lbl5.add_widget(Button(text="ACC", text_size = self.size))

    def clickAction2(self, instance2):
        self.lbl2.text = instance2.text
        #pass

    def clickAction1(self, instance1):
        #print(instance1)
        #identify the button pressed
        t = (1,)
        buttonText = (instance1.text[0:2],)
        print(buttonText)

        #use button text to query all related button info from DB
        c.execute('select * from moderateBaseline where family=?',buttonText)

        #fetch results of query, fetchall() can only be used once
        results = c.fetchall()
        #print(results)
        #print(results[0][3])
        self.lbl2.text = results[1][4]
        #bind root so we can identify on_press in the buttons below
        #self.bind(self)
        # set height of grid before populating with buttons
        self.lbl3.bind(minimum_height=self.lbl3.setter('height'))
        print("lbl3.bind complete")
        # clear all old children
        self.lbl3.clear_widgets()
        print("lbl3 cleared")
        # dynamically add buttons
        for x in results:
            #self.lbl3.add_widget(Button(text=str(x[3])))
            self.lbl3.add_widget(SubControlButtons(text=str(x[3]), theRoot = self))

class MBApp(App):
    # this is a native function from Kivy to actually build the app using KV files
    def build(self):
        #call RootContainer
        return RootContainer()
        # remove the RootContainer call above. Make all the KV file formatting under App
        # (i.e. remove formatting from under RootContainer to one indent left)
        # see ShowcaseApp and how it treats screen organization
        print("RootContainer created")
        root.lbl3.clear_widgets()
        #root.lbl5.add_widget(FamilyButtons(text="AC - Control", theRoot = self, text_size = self.size, on_press = self.clickAction1(self)))

if __name__ == '__main__':
    MBApp().run()<|endoftext|>


<commit_before>import argparse
import json
import os
from glob import glob
from typing import List

import sys

from PIL import Image
from muscima.io import parse_cropobject_list
from tqdm import tqdm

from muscima.cropobject import CropObject

from omrdatasettools.image_generators.ExportPath import ExportPath

class MuscimaPlusPlusImageGenerator:
    def __init__(self) -> None:
        super().__init__()
        self.path_of_this_file = os.path.dirname(os.path.realpath(__file__))

    def extract_and_render_all_symbol_masks(self, raw_data_directory: str, destination_directory: str):
        """
        Extracts all symbols from the raw XML documents and generates individual symbols from the masks

        :param raw_data_directory: The directory, that contains the xml-files and matching images
        :param destination_directory: The directory, in which the symbols should be generated into. One sub-folder per
                                      symbol category will be generated automatically
        """
        print("Extracting Symbols from Muscima++ Dataset...")

        xml_files = self.get_all_xml_file_paths(raw_data_directory)
        crop_objects = self.load_crop_objects_from_xml_files(xml_files)
        self.render_masks_of_crop_objects_into_image(crop_objects, destination_directory)

    def get_all_xml_file_paths(self, raw_data_directory: str) -> List[str]:
        """ Loads all XML-files that are located in the folder.
        :param raw_data_directory: Path to the raw directory, where the MUSCIMA++ dataset was extracted to
        """
        raw_data_directory = os.path.join(raw_data_directory, "v0.9.1", "data", "cropobjects")
        xml_files = [y for x in os.walk(raw_data_directory) for y in glob(os.path.join(x[0], '*.xml'))]
        return xml_files

    def load_crop_objects_from_xml_files(self, xml_files: List[str]) -> List[CropObject]:
        crop_objects = []
        for xml_file in tqdm(xml_files, desc="Loading crop-objects from xml-files", smoothing=0.1, ncols=120):
            crop_objects.extend(self.get_crop_objects_from_xml_file(xml_file))

        for crop_object in crop_objects:
            # Some classes have special characters in their class name that we have to remove
            crop_object.clsname = crop_object.clsname.replace('"', '').replace('/', '').replace('.', '')

        print("Loaded {0} crop-objects".format(len(crop_objects)))
        return crop_objects

    def get_crop_objects_from_xml_file(self, xml_file: str) -> List[CropObject]:
        # e.g., xml_file = 'data/muscima_pp/v0.9/data/cropobjects/CVC-MUSCIMA_W-01_N-10_D-ideal.xml'
        crop_objects = parse_cropobject_list(xml_file)
        return crop_objects

    def render_masks_of_crop_objects_into_image(self, crop_objects: List[CropObject], destination_directory: str):
        for crop_object in tqdm(crop_objects, desc="Generating images from crop-object masks", smoothing=0.1,
                                ncols=120):
            symbol_class = crop_object.clsname
            # Make a copy of the mask to not temper with the original data
            mask = crop_object.mask.copy()
            # We want to draw black symbols on white canvas. The mask encodes foreground pixels
            # that we are interested in with a 1 and background pixels with a 0 and stores those values in
            # an uint8 numpy array. To use Image.fromarray, we have to generate a greyscale mask, where
            # white pixels have the value 255 and black pixels have the value 0. To achieve this, we simply
            # subtract one from each uint, and by exploiting the underflow of the uint we get the following mapping:
            # 0 (background) => 255 (white) and 1 (foreground) => 0 (black) which is exactly what we wanted.
            mask -= 1
            image = Image.fromarray(mask, mode="L")

            target_directory = os.path.join(destination_directory, symbol_class)
            os.makedirs(target_directory, exist_ok=True)

            export_path = ExportPath(destination_directory, symbol_class, crop_object.uid)
            image.save(export_path.get_full_path())

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--raw_dataset_directory",
        type=str,
        default="../data/muscima_pp_raw",
        help="The directory, where the raw Muscima++ dataset can be found")
    parser.add_argument(
        "--image_dataset_directory",
        type=str,
        default="../data/muscima_pp",
        help="The directory, where the generated bitmaps will be created")

    flags, unparsed = parser.parse_known_args()

    muscima_pp_image_generator = MuscimaPlusPlusImageGenerator()
    muscima_pp_image_generator.extract_and_render_all_symbol_masks(flags.raw_dataset_directory,
                                                                   flags.image_dataset_directory)
<commit_msg>Removed fixed column-width from tqdm<commit_after>import argparse
import json
import os
from glob import glob
from typing import List

import sys

from PIL import Image
from muscima.io import parse_cropobject_list
from tqdm import tqdm

from muscima.cropobject import CropObject

from omrdatasettools.image_generators.ExportPath import ExportPath

class MuscimaPlusPlusImageGenerator:
    def __init__(self) -> None:
        super().__init__()
        self.path_of_this_file = os.path.dirname(os.path.realpath(__file__))

    def extract_and_render_all_symbol_masks(self, raw_data_directory: str, destination_directory: str):
        """
        Extracts all symbols from the raw XML documents and generates individual symbols from the masks

        :param raw_data_directory: The directory, that contains the xml-files and matching images
        :param destination_directory: The directory, in which the symbols should be generated into. One sub-folder per
                                      symbol category will be generated automatically
        """
        print("Extracting Symbols from Muscima++ Dataset...")

        xml_files = self.get_all_xml_file_paths(raw_data_directory)
        crop_objects = self.load_crop_objects_from_xml_files(xml_files)
        self.render_masks_of_crop_objects_into_image(crop_objects, destination_directory)

    def get_all_xml_file_paths(self, raw_data_directory: str) -> List[str]:
        """ Loads all XML-files that are located in the folder.
        :param raw_data_directory: Path to the raw directory, where the MUSCIMA++ dataset was extracted to
        """
        raw_data_directory = os.path.join(raw_data_directory, "v0.9.1", "data", "cropobjects")
        xml_files = [y for x in os.walk(raw_data_directory) for y in glob(os.path.join(x[0], '*.xml'))]
        return xml_files

    def load_crop_objects_from_xml_files(self, xml_files: List[str]) -> List[CropObject]:
        crop_objects = []
        for xml_file in tqdm(xml_files, desc="Loading crop-objects from xml-files", smoothing=0.1):
            crop_objects.extend(self.get_crop_objects_from_xml_file(xml_file))

        for crop_object in crop_objects:
            # Some classes have special characters in their class name that we have to remove
            crop_object.clsname = crop_object.clsname.replace('"', '').replace('/', '').replace('.', '')

        print("Loaded {0} crop-objects".format(len(crop_objects)))
        return crop_objects

    def get_crop_objects_from_xml_file(self, xml_file: str) -> List[CropObject]:
        # e.g., xml_file = 'data/muscima_pp/v0.9/data/cropobjects/CVC-MUSCIMA_W-01_N-10_D-ideal.xml'
        crop_objects = parse_cropobject_list(xml_file)
        return crop_objects

    def render_masks_of_crop_objects_into_image(self, crop_objects: List[CropObject], destination_directory: str):
        for crop_object in tqdm(crop_objects, desc="Generating images from crop-object masks", smoothing=0.1):
            symbol_class = crop_object.clsname
            # Make a copy of the mask to not temper with the original data
            mask = crop_object.mask.copy()
            # We want to draw black symbols on white canvas. The mask encodes foreground pixels
            # that we are interested in with a 1 and background pixels with a 0 and stores those values in
            # an uint8 numpy array. To use Image.fromarray, we have to generate a greyscale mask, where
            # white pixels have the value 255 and black pixels have the value 0. To achieve this, we simply
            # subtract one from each uint, and by exploiting the underflow of the uint we get the following mapping:
            # 0 (background) => 255 (white) and 1 (foreground) => 0 (black) which is exactly what we wanted.
            mask -= 1
            image = Image.fromarray(mask, mode="L")

            target_directory = os.path.join(destination_directory, symbol_class)
            os.makedirs(target_directory, exist_ok=True)

            export_path = ExportPath(destination_directory, symbol_class, crop_object.uid)
            image.save(export_path.get_full_path())

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--raw_dataset_directory",
        type=str,
        default="../data/muscima_pp_raw",
        help="The directory, where the raw Muscima++ dataset can be found")
    parser.add_argument(
        "--image_dataset_directory",
        type=str,
        default="../data/muscima_pp",
        help="The directory, where the generated bitmaps will be created")

    flags, unparsed = parser.parse_known_args()

    muscima_pp_image_generator = MuscimaPlusPlusImageGenerator()
    muscima_pp_image_generator.extract_and_render_all_symbol_masks(flags.raw_dataset_directory,
                                                                   flags.image_dataset_directory)
<|endoftext|>


<commit_before>import json

from buffer.models.profile import PATHS, Profile

class Profiles(list):

  def __init__(self, api, *args, **kwargs):
    self.api = api

    super(Profiles, self).__init__(*args, **kwargs)

  def all(self):
    response = self.api.get(url=PATHS['GET_PROFILES'], parser=json.loads)

    for raw_profile in response:
      self.append(Profile(self.api, raw_profile))

    return self

  def filter(self, **kwargs):
    if not len(self):
      self.all()

    new_list = filter(lambda item: [True for arg in kwargs if item[arg] == kwargs[arg]] != [], self)

    return Profiles(self.api, new_list)
<commit_msg>Fix attributes overwritten for profiles<commit_after>import json

from buffer.models.profile import PATHS, Profile

class Profiles(list):

  def __init__(self, api, *args, **kwargs):
    super(Profiles, self).__init__(*args, **kwargs)

    self.api = api

  def all(self):
    response = self.api.get(url=PATHS['GET_PROFILES'], parser=json.loads)

    for raw_profile in response:
      self.append(Profile(self.api, raw_profile))

    return self

  def filter(self, **kwargs):

    if not len(self):
      self.all()

    new_list = filter(lambda item: [True for arg in kwargs if item[arg] == kwargs[arg]] != [], self)

    return Profiles(self.api, new_list)
<|endoftext|>


<commit_before>#Twisted imports
from twisted.internet import ssl, reactor, protocol
from twisted.words.protocols import irc

#system imports
import time, sys, argparse
from ConfigParser import RawConfigParser

class AchievementHandler:
    """
    The class that handles the actual achievements, who gets them, who doesn't, etc.
    """

    def __init__(self, config):
        #TODO: read config file, make settings changes as needed
        pass

    def grant(self, user, achievement):
        #TODO: grant the actual achievement to the user, add the user if they're not in the Users table already
        pass

    def ungrant(self, user, achievement):
        #TODO: remove the achievement from the user
        pass

    def add_achievement(self, name, description=None, requirement=None):
        #TODO: add new descriptions to the database
        pass

    def close(self):
        #TODO: close the connection to the database, finish things out
        pass

class AchieveBot(irc.IRCClient):
    """
    An IRC bot to grant and keep track of achievements users earn.
    """

    nickname = "achievebot"

    #needed methods: connectionMade, connectionLost

    def connectionMade(self):
        irc.IRCClient.connectionMade(self)
        self.achieve = AchievementHandler(open(self.factory.config, 'r'))

    def connectionLost(self, reason):
        irc.IRCClient.connectionLost(self, reason)
        self.achieve.close()

class AchieveBotFactory(protocol.ClientFactory):
    """
    A factory for AchieveBots.
    """

    protocol = AchieveBot

    def __init__(self, config='abot.conf'):
        self.config = config

    def clientConnectionLost(self, connector, reason):
        #reconnect if connection lost
        connector.connect()

    def clientConnectionFailed(self, connector, reason):
        reactor.stop()

if __name__ == '__main__':
    #TODO: get stuff working here
    parser = argparse.ArgumentParser(description='Bot for IRC achievements')
    parser.add_argument('-c', '--config', default='abot.conf', metavar='FILE', help='Config file to use. If it is missing, a default configuration file will be generated at the same path.')
    parser.add_argument('-s', '--server', help='Server to connect to (overrides config file)')
    parser.add_argument('-p', '--port', type=int, help='Port to connect to (overrides config file)')
    parser.add_argument('--ssl', action='store_true', help='Connect using SSL (overrides config file)')
    args = parser.parse_args()

    conf = RawConfigParser()
    try:
        conf.read(args.config)
        serv = conf.get('Connection', 'server')
        port = conf.getint('Connection', 'port')
        usessl = conf.getboolean('Connection', 'usessl')
    except:
        print("Configuration file can't be read. Generating.")
        conf.add_section('Connection')
        conf.set('Connection', 'server', 'INSERT VALUE HERE')
        conf.set('Connection', 'port', '6667')
        conf.set('Connection', 'usessl', 'no')
        conf.write(open(args.config, 'wb'))
        print("Add IRC server address before using.")
        sys.exit()

    if args.server is not None:
        serv = args.server
    if args.port is not None:
        port = args.port
    if args.ssl:
        usessl = True

    f = AchieveBotFactory()
    if usessl:
        reactor.connectSSL(serv, port, f, ssl.CertificateOptions())
    else:
        reactor.connectTCP(serv, port, f)
    reactor.run()

<commit_msg>add commands for quit, join, part<commit_after>#Twisted imports
from twisted.internet import ssl, reactor, protocol
from twisted.words.protocols import irc

#system imports
import time, sys, argparse, csv
from ConfigParser import RawConfigParser

class AchievementHandler:
    """
    The class that handles the actual achievements, who gets them, who doesn't, etc.
    """

    def __init__(self, config):
        #TODO: read config file, make settings changes as needed
        pass

    def command(self, user, channel, msg):
        #TODO: do application-specific command
        pass

    def grant(self, user, achievement):
        #TODO: grant the actual achievement to the user, add the user if they're not in the Users table already
        pass

    def ungrant(self, user, achievement):
        #TODO: remove the achievement from the user
        pass

    def add_achievement(self, name, description=None, requirement=None):
        #TODO: add new descriptions to the database
        pass

    def close(self):
        #TODO: close the connection to the database, finish things out
        pass

class AchieveBot(irc.IRCClient):
    """
    An IRC bot to grant and keep track of achievements users earn.
    """

    nickname = "achievebot"

    #needed methods: connectionMade, connectionLost

    def connectionMade(self):
        irc.IRCClient.connectionMade(self)
        self.achieve = AchievementHandler(open(self.factory.config, 'r'))

    def connectionLost(self, reason):
        irc.IRCClient.connectionLost(self, reason)
        self.achieve.close()

    def privmsg(self, user, channel, msg):
        if channel == self.nickname:
            self.command(user, channel, msg.split())
        elif msg.startswith(self.nickname):
            self.command(user, channel, msg.split()[1:])

    def command(self, user, channel, msg):
        if msg[0] == "quit":
            self.quit(message="I have been told to leave")
        elif msg[0] == "join":
            self.join(msg[1])
        elif msg[0] == "leave":
            self.leave(msg[1], reason="I've been told to part")
        else:
            self.msg(channel, self.achieve.command(msg))

class AchieveBotFactory(protocol.ClientFactory):
    """
    A factory for AchieveBots.
    """

    protocol = AchieveBot

    def __init__(self, config='abot.conf'):
        self.config = config

    def clientConnectionLost(self, connector, reason):
        reactor.stop()
        #reconnect if connection lost
        #connector.connect()

    def clientConnectionFailed(self, connector, reason):
        reactor.stop()

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Bot for IRC achievements')
    parser.add_argument('-c', '--config', default='abot.conf', metavar='FILE', help='Config file to use. If it is missing, a default configuration file will be generated at the same path.')
    parser.add_argument('-s', '--server', help='Server to connect to (overrides config file)')
    parser.add_argument('-p', '--port', type=int, help='Port to connect to (overrides config file)')
    parser.add_argument('--ssl', action='store_true', help='Connect using SSL (overrides config file)')
    args = parser.parse_args()

    conf = RawConfigParser()
    try:
        conf.read(args.config)
        serv = conf.get('Connection', 'server')
        port = conf.getint('Connection', 'port')
        usessl = conf.getboolean('Connection', 'usessl')
    except:
        print("Configuration file can't be read. Generating.")
        conf.add_section('Connection')
        conf.set('Connection', 'server', 'INSERT VALUE HERE')
        conf.set('Connection', 'port', '6667')
        conf.set('Connection', 'usessl', 'no')
        conf.write(open(args.config, 'wb'))
        print("Add IRC server address before using.")
        sys.exit()

    if args.server is not None:
        serv = args.server
    if args.port is not None:
        port = args.port
    if args.ssl:
        usessl = True

    f = AchieveBotFactory()
    if usessl:
        reactor.connectSSL(serv, port, f, ssl.CertificateOptions())
    else:
        reactor.connectTCP(serv, port, f)
    reactor.run()

<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Send a GUI notification to all logged-in users.
# Written by https://github.com/hakavlad .
#
# Example:
#   ./notify_all_users.py earlyoom "Low memory! Killing/Terminating process 2233 tail"
#
# Notification that should pop up:
#   earlyoom
#   Low memory! Killing/Terminating process 2233 tail

from sys import argv
from os import listdir
from subprocess import Popen, TimeoutExpired

if len(argv) < 2 or argv[1] == "-h" or argv[1] == "--help":
    print("Usage:")
    print("  %s [notify-send options] summary [body text]" % (argv[0]))
    print("Examples:")
    print("  %s mytitle mytext" % (argv[0]))
    print("  %s -i dialog-warning earlyoom \"killing process X\"" %
          (argv[0]))
    exit(1)

wait_time = 10

display_env = 'DISPLAY='
dbus_env = 'DBUS_SESSION_BUS_ADDRESS='
user_env = 'USER='

def rline1(path):
    """read 1st line from path."""
    with open(path) as f:
        for line in f:
            return line

def re_pid_environ(pid):
    """
    read environ of 1 process
    returns tuple with USER, DBUS, DISPLAY like follow:
    ('user', 'DISPLAY=:0',
     'DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus')
    returns None if these vars is not in /proc/[pid]/environ
    """
    try:
        env = str(rline1('/proc/' + pid + '/environ'))
        if display_env in env and dbus_env in env and user_env in env:
            env_list = env.split('\x00')

            # iterating over a list of process environment variables
            for i in env_list:
                if i.startswith(user_env):
                    user = i
                    continue

                if i.startswith(display_env):
                    display = i[:10]
                    continue

                if i.startswith(dbus_env):
                    dbus = i
                    continue

                if i.startswith('HOME='):
                    # exclude Display Manager's user
                    if i.startswith('HOME=/var'):
                        return None

            env = user.partition('USER=')[2], display, dbus
            return env

    except FileNotFoundError:
        return None
    except ProcessLookupError:
        return None

def root_notify_env():
    """return set(user, display, dbus)"""
    unsorted_envs_list = []
    # iterates over processes, find processes with suitable env
    for pid in listdir('/proc'):
        if pid[0].isdecimal() is False:
            continue
        one_env = re_pid_environ(pid)
        unsorted_envs_list.append(one_env)
    env = set(unsorted_envs_list)
    env.discard(None)

    # deduplicate dbus
    new_env = []
    end = []
    for i in env:
        key = i[0] + i[1]
        if key not in end:
            end.append(key)
            new_env.append(i)
        else:
            continue

    return new_env

list_with_envs = root_notify_env()

# if somebody logged in with GUI
if len(list_with_envs) > 0:
    # iterating over logged-in users
    for i in list_with_envs:
        username, display_env, dbus_env = i[0], i[1], i[2]
        display_tuple = display_env.partition('=')
        dbus_tuple = dbus_env.partition('=')
        display_key, display_value = display_tuple[0], display_tuple[2]
        dbus_key, dbus_value = dbus_tuple[0], dbus_tuple[2]

        with Popen(['sudo', '-u', username,
                    'notify-send', '--icon=dialog-warning',
                    argv[1], argv[2]
                    ], env={
            display_key: display_value,
            dbus_key: dbus_value
        }) as proc:
            try:
                proc.wait(timeout=wait_time)
            except TimeoutExpired:
                proc.kill()
                print('TimeoutExpired: notify' + username)

else:
    print('Nobody logged-in with GUI. Nothing to do.')
<commit_msg>fix notify helper<commit_after>#!/usr/bin/env python3
#
# Send a GUI notification to all logged-in users.
# Written by https://github.com/hakavlad .
#
# Example:
#   ./notify_all_users.py earlyoom "Low memory! Killing/Terminating process 2233 tail"
#
# Notification that should pop up:
#   earlyoom
#   Low memory! Killing/Terminating process 2233 tail

from sys import argv
from os import listdir
from subprocess import Popen, TimeoutExpired

if len(argv) < 2 or argv[1] == "-h" or argv[1] == "--help":
    print("Usage:")
    print("  %s [notify-send options] summary [body text]" % (argv[0]))
    print("Examples:")
    print("  %s mytitle mytext" % (argv[0]))
    print("  %s -i dialog-warning earlyoom \"killing process X\"" %
          (argv[0]))
    exit(1)

wait_time = 10

display_env = 'DISPLAY='
dbus_env = 'DBUS_SESSION_BUS_ADDRESS='
user_env = 'USER='

def rline1(path):
    """read 1st line from path."""
    with open(path) as f:
        for line in f:
            return line

def re_pid_environ(pid):
    """
    read environ of 1 process
    returns tuple with USER, DBUS, DISPLAY like follow:
    ('user', 'DISPLAY=:0',
     'DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/1000/bus')
    returns None if these vars is not in /proc/[pid]/environ
    """
    try:
        env = str(rline1('/proc/' + pid + '/environ'))
        if display_env in env and dbus_env in env and user_env in env:
            env_list = env.split('\x00')

            # iterating over a list of process environment variables
            for i in env_list:
                if i.startswith(user_env):
                    user = i
                    continue

                if i.startswith(display_env):
                    display = i[:10]
                    continue

                if i.startswith(dbus_env):
                    dbus = i
                    continue

                if i.startswith('HOME='):
                    # exclude Display Manager's user
                    if i.startswith('HOME=/var'):
                        return None

            env = user.partition('USER=')[2], display, dbus
            return env

    except FileNotFoundError:
        return None
    except ProcessLookupError:
        return None

def root_notify_env():
    """return set(user, display, dbus)"""
    unsorted_envs_list = []
    # iterates over processes, find processes with suitable env
    for pid in listdir('/proc'):
        if pid[0].isdecimal() is False:
            continue
        one_env = re_pid_environ(pid)
        unsorted_envs_list.append(one_env)
    env = set(unsorted_envs_list)
    env.discard(None)

    # deduplicate dbus
    new_env = []
    end = []
    for i in env:
        key = i[0] + i[1]
        if key not in end:
            end.append(key)
            new_env.append(i)
        else:
            continue

    return new_env

list_with_envs = root_notify_env()

# if somebody logged in with GUI
if len(list_with_envs) > 0:
    # iterating over logged-in users
    for i in list_with_envs:
        username, dbus_env = i[0], i[2]
        dbus_tuple = dbus_env.partition('=')
        dbus_key, dbus_value = dbus_tuple[0], dbus_tuple[2]

        with Popen([
            'sudo', '-u', username,
            'env', 'DBUS_SESSION_BUS_ADDRESS=' + dbus_value,
            'notify-send', '--icon=dialog-warning', argv[1], argv[2]
        ]) as proc:
            try:
                proc.wait(timeout=wait_time)
            except TimeoutExpired:
                proc.kill()
                print('TimeoutExpired: notify user:' + username)
else:
    print('Nobody logged-in with GUI. Nothing to do.')
<|endoftext|>


<commit_before># python3
# Copyright 2018 DeepMind Technologies Limited. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Common types used throughout Acme."""

from typing import Any, Callable, Iterable, Mapping, Union
from acme import specs

# Define types for nested arrays and tensors.
# TODO(b/144758674): Replace these with recursive type definitions.
NestedArray = Any
NestedTensor = Any

NestedSpec = Union[
    specs.Array,
    Iterable['NestedSpec'],
    Mapping[Any, 'NestedSpec'],  # pytype: disable=not-supported-yet
]

# TODO(b/144763593): Replace all instances of nest with the tensor/array types.
Nest = Union[NestedArray, NestedTensor, NestedSpec]

TensorTransformation = Callable[[NestedTensor], NestedTensor]
TensorValuedCallable = Callable[..., NestedTensor]
<commit_msg>Move Transition class to acme/types.py<commit_after># python3
# Copyright 2018 DeepMind Technologies Limited. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Common types used throughout Acme."""

from typing import Any, Callable, Iterable, Mapping, NamedTuple, Union
from acme import specs

# Define types for nested arrays and tensors.
# TODO(b/144758674): Replace these with recursive type definitions.
NestedArray = Any
NestedTensor = Any

NestedSpec = Union[
    specs.Array,
    Iterable['NestedSpec'],
    Mapping[Any, 'NestedSpec'],  # pytype: disable=not-supported-yet
]

# TODO(b/144763593): Replace all instances of nest with the tensor/array types.
Nest = Union[NestedArray, NestedTensor, NestedSpec]

TensorTransformation = Callable[[NestedTensor], NestedTensor]
TensorValuedCallable = Callable[..., NestedTensor]

class Transition(NamedTuple):
  """Container for a transition."""
  observation: NestedArray
  action: NestedArray
  reward: NestedArray
  discount: NestedArray
  next_observation: NestedArray
<|endoftext|>


<commit_before>#!/usr/bin/python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# expects a Cassandra server to be running and listening on port 9160.
# (read tests expect insert tests to have run first too.)

have_multiproc = False
try:
    from multiprocessing import Array as array, Process as Thread
    from uuid import uuid1 as get_ident
    Thread.isAlive = Thread.is_alive
    have_multiproc = True
except ImportError:
    from threading import Thread
    from thread import get_ident
    from array import array
from hashlib import md5
import time, random, sys, os
from random import randint, gauss
from optparse import OptionParser

from thrift.transport import TTransport
from thrift.transport import TSocket
from thrift.transport import THttpClient
from thrift.protocol import TBinaryProtocol

try:
    from cassandra import Cassandra
    from cassandra.ttypes import *
except ImportError:
    # add cassandra directory to sys.path
    L = os.path.abspath(__file__).split(os.path.sep)[:-3]
    root = os.path.sep.join(L)
    _ipath = os.path.join(root, 'interface', 'thrift', 'gen-py')
    sys.path.append(os.path.join(_ipath, 'cassandra'))
    import Cassandra
    from ttypes import *
except ImportError:
    print "Cassandra thrift bindings not found, please run 'ant gen-thrift-py'"
    sys.exit(2)

try:
    from thrift.protocol import fastbinary
except ImportError:
    print "WARNING: thrift binary extension not found, benchmark will not be accurate!"

parser = OptionParser()
parser.add_option('-n', '--num-keys', type="int", dest="numkeys",
                  help="Number of keys", default=1000**2)
parser.add_option('-t', '--threads', type="int", dest="threads",
                  help="Number of threads/procs to use", default=50)
parser.add_option('-c', '--columns', type="int", dest="columns",
                  help="Number of columns per key", default=5)
parser.add_option('-d', '--nodes', type="string", dest="nodes",
                  help="Host nodes (comma separated)", default="localhost")
parser.add_option('-s', '--stdev', type="int", dest="stdev", default=0.1,
                  help="standard deviation factor")
parser.add_option('-r', '--random', action="store_true", dest="random",
                  help="use random key generator (stdev will have no effect)")
parser.add_option('-f', '--file', type="string", dest="file",
                  help="write output to file")
parser.add_option('-p', '--port', type="int", default=9160, dest="port",
                  help="thrift port")
parser.add_option('-m', '--framed', action="store_true", dest="framed",
                  help="use framed transport")
parser.add_option('-o', '--operation', type="choice", dest="operation",
                  default="insert", choices=('insert', 'read', 'rangeslice'),
                  help="operation to perform")
parser.add_option('-u', '--supercolumns', type="int", dest="supers", default=1,
                  help="number of super columns per key")
parser.add_option('-y', '--family-type', type="choice", dest="cftype",
                  choices=('regular','super'), default='regular',
                  help="column family type")
parser.add_option('-k', '--keep-going', action="store_true", dest="ignore",
                  help="ignore errors inserting or reading")
parser.add_option('-i', '--progress-interval', type="int", default=10,
                  dest="interval", help="progress report interval (seconds)")
parser.add_option('-g', '--get-range-slice-count', type="int", default=1000,
                  dest="rangecount",
                  help="amount of keys to get_range_slices per call")

(options, args) = parser.parse_args()

total_keys = options.numkeys
n_threads = options.threads
keys_per_thread = total_keys / n_threads
columns_per_key = options.columns
supers_per_key = options.supers
# this allows client to round robin requests directly for
# simple request load-balancing
nodes = options.nodes.split(',')

# a generator that generates all keys according to a bell curve centered
# around the middle of the keys generated (0..total_keys).  Remember that
# about 68% of keys will be within stdev away from the mean and
# about 95% within 2*stdev.
stdev = total_keys * options.stdev
mean = total_keys / 2

def key_generator_gauss():
    fmt = '%0' + str(len(str(total_keys))) + 'd'
    while True:
        guess = gauss(mean, stdev)
        if 0 <= guess < total_keys:
            return fmt % int(guess)

# a generator that will generate all keys w/ equal probability.  this is the
# worst case for caching.
def key_generator_random():
    fmt = '%0' + str(len(str(total_keys))) + 'd'
    return fmt % randint(0, total_keys - 1)

key_generator = key_generator_gauss
if options.random:
    key_generator = key_generator_random

def get_client(host='127.0.0.1', port=9160, framed=False):
    socket = TSocket.TSocket(host, port)
    if framed:
        transport = TTransport.TFramedTransport(socket)
    else:
        transport = TTransport.TBufferedTransport(socket)
    protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
    client = Cassandra.Client(protocol)
    client.transport = transport
    return client

def make_keyspaces():
    cfams = [CfDef('Keyspace1', 'Standard1'),
             CfDef('Keyspace1', 'Super1', 'Super')]
    keyspace = KsDef('Keyspace1', 'org.apache.cassandra.locator.RackUnawareStrategy', 1, cfams)
    client = get_client(nodes[0], options.port, options.framed)
    client.transport.open()
    try:
        client.system_add_keyspace(keyspace)
        print "Created keyspaces.  Sleeping %ss for propagation." % len(nodes)
        time.sleep(len(nodes))
    except InvalidRequestException, e:
        print e.why
    client.transport.close()

class Operation(Thread):
    def __init__(self, i, opcounts, keycounts, latencies):
        Thread.__init__(self)
        # generator of the keys to be used
        self.range = xrange(keys_per_thread * i, keys_per_thread * (i + 1))
        # we can't use a local counter, since that won't be visible to the parent
        # under multiprocessing.  instead, the parent passes a "opcounts" array
        # and an index that is our assigned counter.
        self.idx = i
        self.opcounts = opcounts
        # similarly, a shared array for latency and key totals
        self.latencies = latencies
        self.keycounts = keycounts
        # random host for pseudo-load-balancing
        [hostname] = random.sample(nodes, 1)
        # open client
        self.cclient = get_client(hostname, options.port, options.framed)
        self.cclient.transport.open()
        self.cclient.set_keyspace('Keyspace1')

class Inserter(Operation):
    def run(self):
        data = md5(str(get_ident())).hexdigest()
        columns = [Column('C' + str(j), data, Clock(0)) for j in xrange(columns_per_key)]
        fmt = '%0' + str(len(str(total_keys))) + 'd'
        if 'super' == options.cftype:
            supers = [SuperColumn('S' + str(j), columns) for j in xrange(supers_per_key)]
        for i in self.range:
            key = fmt % i
            if 'super' == options.cftype:
                cfmap= {key: {'Super1' : [Mutation(ColumnOrSuperColumn(super_column=s)) for s in supers]}}
            else:
                cfmap = {key: {'Standard1': [Mutation(ColumnOrSuperColumn(column=c)) for c in columns]}}
            start = time.time()
            try:
                self.cclient.batch_mutate(cfmap, ConsistencyLevel.ONE)
            except KeyboardInterrupt:
                raise
            except Exception, e:
                if options.ignore:
                    print e
                else:
                    raise
            self.latencies[self.idx] += time.time() - start
            self.opcounts[self.idx] += 1
            self.keycounts[self.idx] += 1

class Reader(Operation):
    def run(self):
        p = SlicePredicate(slice_range=SliceRange('', '', False, columns_per_key))
        if 'super' == options.cftype:
            for i in xrange(keys_per_thread):
                key = key_generator()
                for j in xrange(supers_per_key):
                    parent = ColumnParent('Super1', 'S' + str(j))
                    start = time.time()
                    try:
                        r = self.cclient.get_slice(key, parent, p, ConsistencyLevel.ONE)
                        if not r: raise RuntimeError("Key %s not found" % key)
                    except KeyboardInterrupt:
                        raise
                    except Exception, e:
                        if options.ignore:
                            print e
                        else:
                            raise
                    self.latencies[self.idx] += time.time() - start
                    self.opcounts[self.idx] += 1
                    self.keycounts[self.idx] += 1
        else:
            parent = ColumnParent('Standard1')
            for i in xrange(keys_per_thread):
                key = key_generator()
                start = time.time()
                try:
                    r = self.cclient.get_slice(key, parent, p, ConsistencyLevel.ONE)
                    if not r: raise RuntimeError("Key %s not found" % key)
                except KeyboardInterrupt:
                    raise
                except Exception, e:
                    if options.ignore:
                        print e
                    else:
                        raise
                self.latencies[self.idx] += time.time() - start
                self.opcounts[self.idx] += 1
                self.keycounts[self.idx] += 1

class RangeSlicer(Operation):
    def run(self):
        begin = self.range[0]
        end = self.range[-1]
        current = begin
        last = current + options.rangecount
        fmt = '%0' + str(len(str(total_keys))) + 'd'
        p = SlicePredicate(slice_range=SliceRange('', '', False, columns_per_key))
        if 'super' == options.cftype:
            while current < end:
                keyrange = KeyRange(fmt % current, fmt % last, count = options.rangecount)
                res = []
                for j in xrange(supers_per_key):
                    parent = ColumnParent('Super1', chr(ord('A') + j))
                    begin = time.time()
                    try:
                        res = self.cclient.get_range_slices(parent, p, keyrange, ConsistencyLevel.ONE)
                        if not res: raise RuntimeError("Key %s not found" % key)
                    except KeyboardInterrupt:
                        raise
                    except Exception, e:
                        if options.ignore:
                            print e
                        else:
                            raise
                    self.latencies[self.idx] += time.time() - begin
                    self.opcounts[self.idx] += 1
                current += len(r) + 1
                last = current + len(r) + 1
                self.keycounts[self.idx] += len(r)
        else:
            parent = ColumnParent('Standard1')
            while current < end:
                start = fmt % current
                finish = fmt % last
                keyrange = KeyRange(start, finish, count = options.rangecount)
                begin = time.time()
                try:
                    r = self.cclient.get_range_slices(parent, p, keyrange, ConsistencyLevel.ONE)
                    if not r: raise RuntimeError("Range not found:", start, finish)
                except KeyboardInterrupt:
                    raise
                except Exception, e:
                    if options.ignore:
                        print e
                    else:
                        print start, finish
                        raise
                current += len(r) + 1
                last = current + len(r)  + 1
                self.latencies[self.idx] += time.time() - begin
                self.opcounts[self.idx] += 1
                self.keycounts[self.idx] += len(r)

class OperationFactory:
    @staticmethod
    def create(type, i, opcounts, keycounts, latencies):
        if type == 'read':
            return Reader(i, opcounts, keycounts, latencies)
        elif type == 'insert':
            return Inserter(i, opcounts, keycounts, latencies)
        elif type == 'rangeslice':
            return RangeSlicer(i, opcounts, keycounts, latencies)
        else:
            raise RuntimeError, 'Unsupported op!'

class Stress(object):
    opcounts = array('i', [0] * n_threads)
    latencies = array('d', [0] * n_threads)
    keycounts = array('i', [0] * n_threads)

    def create_threads(self,type):
        threads = []
        for i in xrange(n_threads):
            th = OperationFactory.create(type, i, self.opcounts, self.keycounts, self.latencies)
            threads.append(th)
            th.start()
        return threads

    def run_test(self,filename,threads):
        start_t = time.time()
        if filename:
            outf = open(filename,'w')
        else:
            outf = sys.stdout
        outf.write('total,interval_op_rate,interval_key_rate,avg_latency,elapsed_time\n')
        epoch = total = old_total = latency = keycount = old_keycount = old_latency = 0
        epoch_intervals = (options.interval * 10) # 1 epoch = 1 tenth of a second
        terminate = False
        while not terminate:
            time.sleep(0.1)
            if not [th for th in threads if th.isAlive()]:
                terminate = True
            epoch = epoch + 1
            if terminate or epoch > epoch_intervals:
                epoch = 0
                old_total, old_latency, old_keycount = total, latency, keycount
                total = sum(self.opcounts[th.idx] for th in threads)
                latency = sum(self.latencies[th.idx] for th in threads)
                keycount = sum(self.keycounts[th.idx] for th in threads)
                opdelta = total - old_total
                keydelta = keycount - old_keycount
                delta_latency = latency - old_latency
                if opdelta > 0:
                    delta_formatted = (delta_latency / opdelta)
                else:
                    delta_formatted = 'NaN'
                elapsed_t = int(time.time() - start_t)
                outf.write('%d,%d,%d,%s,%d\n'
                           % (total, opdelta / options.interval, keydelta / options.interval, delta_formatted, elapsed_t))

    def insert(self):
        threads = self.create_threads('insert')
        self.run_test(options.file,threads);

    def read(self):
        threads = self.create_threads('read')
        self.run_test(options.file,threads);

    def rangeslice(self):
        threads = self.create_threads('rangeslice')
        self.run_test(options.file,threads);

stresser = Stress()
benchmark = getattr(stresser, options.operation, None)
if not have_multiproc:
    print """WARNING: multiprocessing not present, threading will be used.
        Benchmark may not be accurate!"""
if options.operation == 'insert':
    make_keyspaces()
benchmark()
<commit_msg>stress.py stdev accepts float instead of int.  Patch by Oren Benjamin; revied by brandonwilliams for CASSANDRA-1262<commit_after>#!/usr/bin/python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# expects a Cassandra server to be running and listening on port 9160.
# (read tests expect insert tests to have run first too.)

have_multiproc = False
try:
    from multiprocessing import Array as array, Process as Thread
    from uuid import uuid1 as get_ident
    Thread.isAlive = Thread.is_alive
    have_multiproc = True
except ImportError:
    from threading import Thread
    from thread import get_ident
    from array import array
from hashlib import md5
import time, random, sys, os
from random import randint, gauss
from optparse import OptionParser

from thrift.transport import TTransport
from thrift.transport import TSocket
from thrift.transport import THttpClient
from thrift.protocol import TBinaryProtocol

try:
    from cassandra import Cassandra
    from cassandra.ttypes import *
except ImportError:
    # add cassandra directory to sys.path
    L = os.path.abspath(__file__).split(os.path.sep)[:-3]
    root = os.path.sep.join(L)
    _ipath = os.path.join(root, 'interface', 'thrift', 'gen-py')
    sys.path.append(os.path.join(_ipath, 'cassandra'))
    import Cassandra
    from ttypes import *
except ImportError:
    print "Cassandra thrift bindings not found, please run 'ant gen-thrift-py'"
    sys.exit(2)

try:
    from thrift.protocol import fastbinary
except ImportError:
    print "WARNING: thrift binary extension not found, benchmark will not be accurate!"

parser = OptionParser()
parser.add_option('-n', '--num-keys', type="int", dest="numkeys",
                  help="Number of keys", default=1000**2)
parser.add_option('-t', '--threads', type="int", dest="threads",
                  help="Number of threads/procs to use", default=50)
parser.add_option('-c', '--columns', type="int", dest="columns",
                  help="Number of columns per key", default=5)
parser.add_option('-d', '--nodes', type="string", dest="nodes",
                  help="Host nodes (comma separated)", default="localhost")
parser.add_option('-s', '--stdev', type="float", dest="stdev", default=0.1,
                  help="standard deviation factor")
parser.add_option('-r', '--random', action="store_true", dest="random",
                  help="use random key generator (stdev will have no effect)")
parser.add_option('-f', '--file', type="string", dest="file",
                  help="write output to file")
parser.add_option('-p', '--port', type="int", default=9160, dest="port",
                  help="thrift port")
parser.add_option('-m', '--framed', action="store_true", dest="framed",
                  help="use framed transport")
parser.add_option('-o', '--operation', type="choice", dest="operation",
                  default="insert", choices=('insert', 'read', 'rangeslice'),
                  help="operation to perform")
parser.add_option('-u', '--supercolumns', type="int", dest="supers", default=1,
                  help="number of super columns per key")
parser.add_option('-y', '--family-type', type="choice", dest="cftype",
                  choices=('regular','super'), default='regular',
                  help="column family type")
parser.add_option('-k', '--keep-going', action="store_true", dest="ignore",
                  help="ignore errors inserting or reading")
parser.add_option('-i', '--progress-interval', type="int", default=10,
                  dest="interval", help="progress report interval (seconds)")
parser.add_option('-g', '--get-range-slice-count', type="int", default=1000,
                  dest="rangecount",
                  help="amount of keys to get_range_slices per call")

(options, args) = parser.parse_args()

total_keys = options.numkeys
n_threads = options.threads
keys_per_thread = total_keys / n_threads
columns_per_key = options.columns
supers_per_key = options.supers
# this allows client to round robin requests directly for
# simple request load-balancing
nodes = options.nodes.split(',')

# a generator that generates all keys according to a bell curve centered
# around the middle of the keys generated (0..total_keys).  Remember that
# about 68% of keys will be within stdev away from the mean and
# about 95% within 2*stdev.
stdev = total_keys * options.stdev
mean = total_keys / 2

def key_generator_gauss():
    fmt = '%0' + str(len(str(total_keys))) + 'd'
    while True:
        guess = gauss(mean, stdev)
        if 0 <= guess < total_keys:
            return fmt % int(guess)

# a generator that will generate all keys w/ equal probability.  this is the
# worst case for caching.
def key_generator_random():
    fmt = '%0' + str(len(str(total_keys))) + 'd'
    return fmt % randint(0, total_keys - 1)

key_generator = key_generator_gauss
if options.random:
    key_generator = key_generator_random

def get_client(host='127.0.0.1', port=9160, framed=False):
    socket = TSocket.TSocket(host, port)
    if framed:
        transport = TTransport.TFramedTransport(socket)
    else:
        transport = TTransport.TBufferedTransport(socket)
    protocol = TBinaryProtocol.TBinaryProtocolAccelerated(transport)
    client = Cassandra.Client(protocol)
    client.transport = transport
    return client

def make_keyspaces():
    cfams = [CfDef('Keyspace1', 'Standard1'),
             CfDef('Keyspace1', 'Super1', 'Super')]
    keyspace = KsDef('Keyspace1', 'org.apache.cassandra.locator.RackUnawareStrategy', 1, cfams)
    client = get_client(nodes[0], options.port, options.framed)
    client.transport.open()
    try:
        client.system_add_keyspace(keyspace)
        print "Created keyspaces.  Sleeping %ss for propagation." % len(nodes)
        time.sleep(len(nodes))
    except InvalidRequestException, e:
        print e.why
    client.transport.close()

class Operation(Thread):
    def __init__(self, i, opcounts, keycounts, latencies):
        Thread.__init__(self)
        # generator of the keys to be used
        self.range = xrange(keys_per_thread * i, keys_per_thread * (i + 1))
        # we can't use a local counter, since that won't be visible to the parent
        # under multiprocessing.  instead, the parent passes a "opcounts" array
        # and an index that is our assigned counter.
        self.idx = i
        self.opcounts = opcounts
        # similarly, a shared array for latency and key totals
        self.latencies = latencies
        self.keycounts = keycounts
        # random host for pseudo-load-balancing
        [hostname] = random.sample(nodes, 1)
        # open client
        self.cclient = get_client(hostname, options.port, options.framed)
        self.cclient.transport.open()
        self.cclient.set_keyspace('Keyspace1')

class Inserter(Operation):
    def run(self):
        data = md5(str(get_ident())).hexdigest()
        columns = [Column('C' + str(j), data, Clock(0)) for j in xrange(columns_per_key)]
        fmt = '%0' + str(len(str(total_keys))) + 'd'
        if 'super' == options.cftype:
            supers = [SuperColumn('S' + str(j), columns) for j in xrange(supers_per_key)]
        for i in self.range:
            key = fmt % i
            if 'super' == options.cftype:
                cfmap= {key: {'Super1' : [Mutation(ColumnOrSuperColumn(super_column=s)) for s in supers]}}
            else:
                cfmap = {key: {'Standard1': [Mutation(ColumnOrSuperColumn(column=c)) for c in columns]}}
            start = time.time()
            try:
                self.cclient.batch_mutate(cfmap, ConsistencyLevel.ONE)
            except KeyboardInterrupt:
                raise
            except Exception, e:
                if options.ignore:
                    print e
                else:
                    raise
            self.latencies[self.idx] += time.time() - start
            self.opcounts[self.idx] += 1
            self.keycounts[self.idx] += 1

class Reader(Operation):
    def run(self):
        p = SlicePredicate(slice_range=SliceRange('', '', False, columns_per_key))
        if 'super' == options.cftype:
            for i in xrange(keys_per_thread):
                key = key_generator()
                for j in xrange(supers_per_key):
                    parent = ColumnParent('Super1', 'S' + str(j))
                    start = time.time()
                    try:
                        r = self.cclient.get_slice(key, parent, p, ConsistencyLevel.ONE)
                        if not r: raise RuntimeError("Key %s not found" % key)
                    except KeyboardInterrupt:
                        raise
                    except Exception, e:
                        if options.ignore:
                            print e
                        else:
                            raise
                    self.latencies[self.idx] += time.time() - start
                    self.opcounts[self.idx] += 1
                    self.keycounts[self.idx] += 1
        else:
            parent = ColumnParent('Standard1')
            for i in xrange(keys_per_thread):
                key = key_generator()
                start = time.time()
                try:
                    r = self.cclient.get_slice(key, parent, p, ConsistencyLevel.ONE)
                    if not r: raise RuntimeError("Key %s not found" % key)
                except KeyboardInterrupt:
                    raise
                except Exception, e:
                    if options.ignore:
                        print e
                    else:
                        raise
                self.latencies[self.idx] += time.time() - start
                self.opcounts[self.idx] += 1
                self.keycounts[self.idx] += 1

class RangeSlicer(Operation):
    def run(self):
        begin = self.range[0]
        end = self.range[-1]
        current = begin
        last = current + options.rangecount
        fmt = '%0' + str(len(str(total_keys))) + 'd'
        p = SlicePredicate(slice_range=SliceRange('', '', False, columns_per_key))
        if 'super' == options.cftype:
            while current < end:
                keyrange = KeyRange(fmt % current, fmt % last, count = options.rangecount)
                res = []
                for j in xrange(supers_per_key):
                    parent = ColumnParent('Super1', chr(ord('A') + j))
                    begin = time.time()
                    try:
                        res = self.cclient.get_range_slices(parent, p, keyrange, ConsistencyLevel.ONE)
                        if not res: raise RuntimeError("Key %s not found" % key)
                    except KeyboardInterrupt:
                        raise
                    except Exception, e:
                        if options.ignore:
                            print e
                        else:
                            raise
                    self.latencies[self.idx] += time.time() - begin
                    self.opcounts[self.idx] += 1
                current += len(r) + 1
                last = current + len(r) + 1
                self.keycounts[self.idx] += len(r)
        else:
            parent = ColumnParent('Standard1')
            while current < end:
                start = fmt % current
                finish = fmt % last
                keyrange = KeyRange(start, finish, count = options.rangecount)
                begin = time.time()
                try:
                    r = self.cclient.get_range_slices(parent, p, keyrange, ConsistencyLevel.ONE)
                    if not r: raise RuntimeError("Range not found:", start, finish)
                except KeyboardInterrupt:
                    raise
                except Exception, e:
                    if options.ignore:
                        print e
                    else:
                        print start, finish
                        raise
                current += len(r) + 1
                last = current + len(r)  + 1
                self.latencies[self.idx] += time.time() - begin
                self.opcounts[self.idx] += 1
                self.keycounts[self.idx] += len(r)

class OperationFactory:
    @staticmethod
    def create(type, i, opcounts, keycounts, latencies):
        if type == 'read':
            return Reader(i, opcounts, keycounts, latencies)
        elif type == 'insert':
            return Inserter(i, opcounts, keycounts, latencies)
        elif type == 'rangeslice':
            return RangeSlicer(i, opcounts, keycounts, latencies)
        else:
            raise RuntimeError, 'Unsupported op!'

class Stress(object):
    opcounts = array('i', [0] * n_threads)
    latencies = array('d', [0] * n_threads)
    keycounts = array('i', [0] * n_threads)

    def create_threads(self,type):
        threads = []
        for i in xrange(n_threads):
            th = OperationFactory.create(type, i, self.opcounts, self.keycounts, self.latencies)
            threads.append(th)
            th.start()
        return threads

    def run_test(self,filename,threads):
        start_t = time.time()
        if filename:
            outf = open(filename,'w')
        else:
            outf = sys.stdout
        outf.write('total,interval_op_rate,interval_key_rate,avg_latency,elapsed_time\n')
        epoch = total = old_total = latency = keycount = old_keycount = old_latency = 0
        epoch_intervals = (options.interval * 10) # 1 epoch = 1 tenth of a second
        terminate = False
        while not terminate:
            time.sleep(0.1)
            if not [th for th in threads if th.isAlive()]:
                terminate = True
            epoch = epoch + 1
            if terminate or epoch > epoch_intervals:
                epoch = 0
                old_total, old_latency, old_keycount = total, latency, keycount
                total = sum(self.opcounts[th.idx] for th in threads)
                latency = sum(self.latencies[th.idx] for th in threads)
                keycount = sum(self.keycounts[th.idx] for th in threads)
                opdelta = total - old_total
                keydelta = keycount - old_keycount
                delta_latency = latency - old_latency
                if opdelta > 0:
                    delta_formatted = (delta_latency / opdelta)
                else:
                    delta_formatted = 'NaN'
                elapsed_t = int(time.time() - start_t)
                outf.write('%d,%d,%d,%s,%d\n'
                           % (total, opdelta / options.interval, keydelta / options.interval, delta_formatted, elapsed_t))

    def insert(self):
        threads = self.create_threads('insert')
        self.run_test(options.file,threads);

    def read(self):
        threads = self.create_threads('read')
        self.run_test(options.file,threads);

    def rangeslice(self):
        threads = self.create_threads('rangeslice')
        self.run_test(options.file,threads);

stresser = Stress()
benchmark = getattr(stresser, options.operation, None)
if not have_multiproc:
    print """WARNING: multiprocessing not present, threading will be used.
        Benchmark may not be accurate!"""
if options.operation == 'insert':
    make_keyspaces()
benchmark()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# You must disable this pipeline in settings.py to run on Scrapy Cloud

from items import course_item
import datetime
import json

import os
import threading

from scrapinghub.hubstorage.serialization import jsondefault
from scrapinghub.hubstorage.utils import millitime

class NesappscraperPipelineScrapyCloud(object):
    # from https://github.com/scrapinghub/scrapinghub-entrypoint-scrapy/blob/master/sh_scrapy/writer.py
    def __init__(self):
        self.path = path or ''
        self._lock = threading.Lock()
        self._pipe = None
        if not self.path:
            self._write = _not_configured
            self.open = _not_configured
            self.close = _not_configured

    # called when the spider opens
    def open_spider(self, spider):
        # open pipe
        with self._lock:
            self._pipe = open(self.path, 'wb')
        # create list of exam packs
        self.exam_pack_list = []
        # creates counter
        self.count = 0
        # gets current time
        self.start_time = datetime.datetime.today()

    def _write(self, command, payload):
        # binary command
        command = command.encode('utf-8')
        # binary payload
        encoded_payload = json.dumps(
            payload,
            separators=(',', ':'),
            default=jsondefault
        ).encode('utf-8')
        # write needs to be locked because write can be called from multiple threads
        with self._lock:
            self._pipe.write(command)
            self._pipe.write(b' ')
            self._pipe.write(encoded_payload)
            self._pipe.write(b'\n')
            self._pipe.flush()

    def write_item(self, item):
        self._write('ITM', item)

    def write_stats(self, stats):
        self._write('STA', {'time': millitime(), 'stats': stats})

    # called when each exam_pack_item is yielded in parse_docs
    def process_item(self, item, spider):
        # increment counter
        self.count += 1
        # flag if exam pack was grouped with other exam packs of the same course
        placed = False
        # loops through each element in the list to see if an exam pack has
        # already been stored for this course
        for i in range(len(self.exam_pack_list)):
            course_name = self.exam_pack_list[i][0]['course']
            if item['course'] == course_name:
                # group this exam pack with other exam packs of the same course
                # already in the list
                self.exam_pack_list[i].append(item)
                placed = True
        # If this is the first exam pack of the course, append it to the lsit
        if placed == False:
            self.exam_pack_list.append([item])
        # Returns nothing - usually this would return an item
        # This prevents it from working on scrapy cloud
        return None

    # called when the spider closes
    def close_spider(self, spider):
        # Loops through each course in exam_pack_list
        for i in range(len(self.exam_pack_list)):
            # Makes a course_item for each course and formats properly
            line = course_item(
                course_name = self.exam_pack_list[i][0]['course'],
                packs = self.exam_pack_list[i]
            )
            # Writes coruse_item to the JSON file
            self.write_item(line)

        # Get end time
        self.end_time = datetime.datetime.today()
        # Get runtime
        self.runtime = self.end_time - self.start_time

        # Writes to meta json file
        self.write_stats(
            'STA { "stats: {' +
            '"timestamp": "' + str( self.end_time.isoformat() ) + '", ' +
            '"runtime": "' + str( self.runtime.total_seconds() ) + '", ' +
            '"exam_pack_items": ' + str( self.count ) +
            ' }}'
        )

        # finished!
        self._write('FIN', {'outcome': 'finished'})
        self._pipe.flush()
        self._pipe.close()
<commit_msg>cloudtest<commit_after># -*- coding: utf-8 -*-

# You must disable this pipeline in settings.py to run on Scrapy Cloud

from items import course_item
import datetime
import json

import os
import threading

from scrapinghub.hubstorage.serialization import jsondefault
from scrapinghub.hubstorage.utils import millitime

class NesappscraperPipelineScrapyCloud(object):
    # from https://github.com/scrapinghub/scrapinghub-entrypoint-scrapy/blob/master/sh_scrapy/writer.py
    def __init__(self):
        self.path = ''
        self._lock = threading.Lock()
        self._pipe = None
        if not self.path:
            self._write = _not_configured
            self.open = _not_configured
            self.close = _not_configured

    # called when the spider opens
    def open_spider(self, spider):
        # open pipe
        with self._lock:
            self._pipe = open(self.path, 'wb')
        # create list of exam packs
        self.exam_pack_list = []
        # creates counter
        self.count = 0
        # gets current time
        self.start_time = datetime.datetime.today()

    def _write(self, command, payload):
        # binary command
        command = command.encode('utf-8')
        # binary payload
        encoded_payload = json.dumps(
            payload,
            separators=(',', ':'),
            default=jsondefault
        ).encode('utf-8')
        # write needs to be locked because write can be called from multiple threads
        with self._lock:
            self._pipe.write(command)
            self._pipe.write(b' ')
            self._pipe.write(encoded_payload)
            self._pipe.write(b'\n')
            self._pipe.flush()

    def write_item(self, item):
        self._write('ITM', item)

    def write_stats(self, stats):
        self._write('STA', {'time': millitime(), 'stats': stats})

    # called when each exam_pack_item is yielded in parse_docs
    def process_item(self, item, spider):
        # increment counter
        self.count += 1
        # flag if exam pack was grouped with other exam packs of the same course
        placed = False
        # loops through each element in the list to see if an exam pack has
        # already been stored for this course
        for i in range(len(self.exam_pack_list)):
            course_name = self.exam_pack_list[i][0]['course']
            if item['course'] == course_name:
                # group this exam pack with other exam packs of the same course
                # already in the list
                self.exam_pack_list[i].append(item)
                placed = True
        # If this is the first exam pack of the course, append it to the lsit
        if placed == False:
            self.exam_pack_list.append([item])
        # Returns nothing - usually this would return an item
        # This prevents it from working on scrapy cloud
        return None

    # called when the spider closes
    def close_spider(self, spider):
        # Loops through each course in exam_pack_list
        for i in range(len(self.exam_pack_list)):
            # Makes a course_item for each course and formats properly
            line = course_item(
                course_name = self.exam_pack_list[i][0]['course'],
                packs = self.exam_pack_list[i]
            )
            # Writes coruse_item to the JSON file
            self.write_item(line)

        # Get end time
        self.end_time = datetime.datetime.today()
        # Get runtime
        self.runtime = self.end_time - self.start_time

        # Writes to meta json file
        self.write_stats(
            'STA { "stats: {' +
            '"timestamp": "' + str( self.end_time.isoformat() ) + '", ' +
            '"runtime": "' + str( self.runtime.total_seconds() ) + '", ' +
            '"exam_pack_items": ' + str( self.count ) +
            ' }}'
        )

        # finished!
        self._write('FIN', {'outcome': 'finished'})
        self._pipe.flush()
        self._pipe.close()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

# Django imports
from django.apps import AppConfig

class DevilryQualifiesForExamApprovedAppConfig(AppConfig):
    name = 'devilry.devilry_qualifiesforexam_plugin_approved'
    verbose_name = 'Devilry qualifies for exam plugin approved'

    def ready(self):
        print 'Loading approved plugin'
        from devilry.devilry_qualifiesforexam import plugintyperegistry
        from . import plugin
        plugintyperegistry.Registry.get_instance().add(plugin.SelectAssignmentsPlugin)
<commit_msg>devilry_qualifiesforexam_plugin_approved/apps.py: removed load print statement.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

# Django imports
from django.apps import AppConfig

class DevilryQualifiesForExamApprovedAppConfig(AppConfig):
    name = 'devilry.devilry_qualifiesforexam_plugin_approved'
    verbose_name = 'Devilry qualifies for exam plugin approved'

    def ready(self):
        from devilry.devilry_qualifiesforexam import plugintyperegistry
        from . import plugin
        plugintyperegistry.Registry.get_instance().add(plugin.SelectAssignmentsPlugin)
<|endoftext|>


<commit_before>"""
    Recreate the trajectory fragments to the led to the discovery of a snapshot,
    specified by the tuple (epoch, trajectory, snapshot) and write as a pdb file
"""
from __future__ import print_function
import os
import sys
import argparse
import glob
import itertools
from AdaptivePELE.utilities import utilities
from AdaptivePELE.atomset import atomset
try:
    basestring
except NameError:
    basestring = str

def parseArguments():
    """
        Parse the command-line options

        :returns: :py:class:`.Clustering`, int, int, int, str -- Clustering
            object, number of trajectory, number of snapshot, number of epoch,
            output path where to write the files
    """
    desc = "Write the information related to the conformation network to file\n"
    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument("epoch", type=str, help="Path to the epoch to search the snapshot")
    parser.add_argument("trajectory", type=int, help="Trajectory number")
    parser.add_argument("snapshot", type=int, help="Snapshot to select (in accepted steps)")
    parser.add_argument("-o", type=str, default=None, help="Output path where to write the files")
    parser.add_argument("--name", type=str, default="pathway.pdb", help="Name of the pdb to write the files")
    parser.add_argument("--top", type=str, default=None, help="Name of the pdb topology for loading non-pdb trajectories")
    args = parser.parse_args()
    return args.trajectory, args.snapshot, args.epoch, args.o, args.name, args.top

def main(trajectory, snapshot, epoch, outputPath, out_filename, topology):
    if outputPath is not None:
        outputPath = os.path.join(outputPath, "")
        if not os.path.exists(outputPath):
            os.makedirs(outputPath)
    else:
        outputPath = ""
    if topology is not None:
        topology_contents = utilities.getTopologyFile(topology)
    else:
        topology_contents = None
    if os.path.exists(outputPath+out_filename):
        # If the specified name exists, append a number to distinguish the files
        name, ext = os.path.splitext(out_filename)
        out_filename = "".join([name, "_%d", ext])
        i = 1
        while os.path.exists(outputPath+out_filename % i):
            i += 1
        out_filename %= i
    pathway = []
    # Strip out trailing backslash if present
    pathPrefix, epoch = os.path.split(epoch.rstrip("/"))
    sys.stderr.write("Creating pathway...\n")
    while True:
        filename = glob.glob(os.path.join(pathPrefix, epoch, "*traj*_%d*" % trajectory))
        snapshots = utilities.getSnapshots(filename[0], topology=topology)
        if not isinstance(snapshots[0], basestring):
            new_snapshots = []
            for snapshot in snapshots[:snapshots+1]:
                PDB = atomset.PDB()
                PDB.initialise(snapshot)
                new_snapshots.append(PDB.get_pdb_string(topology_contents))
            snapshots = new_snapshots
        else:
            snapshots = snapshots[:snapshot+1]
        pathway.insert(0, snapshots)
        if epoch == '0':
            # Once we get to epoch 0, we just need to append the trajectory
            # where the cluster was found and we can break out of the loop
            break
        procMapping = open(os.path.join(pathPrefix, epoch, "processorMapping.txt")).read().rstrip().split(':')
        epoch, trajectory, snapshot = map(int, procMapping[trajectory-1][1:-1].split(','))
        epoch = str(epoch)
    sys.stderr.write("Writing pathway...\n")
    with open(outputPath+out_filename, "a") as f:
        f.write("ENDMDL\n".join(itertools.chain.from_iterable(pathway)))

if __name__ == "__main__":
    traj, num_snapshot, num_epoch, output_path, output_filename, top = parseArguments()
    main(traj, num_snapshot, num_epoch, output_path, output_filename, top)
<commit_msg>Fix bugs in backtrackAdaptiveTrajectory for xtc<commit_after>"""
    Recreate the trajectory fragments to the led to the discovery of a snapshot,
    specified by the tuple (epoch, trajectory, snapshot) and write as a pdb file
"""
from __future__ import print_function
import os
import sys
import argparse
import glob
import itertools
from AdaptivePELE.utilities import utilities
from AdaptivePELE.atomset import atomset
try:
    basestring
except NameError:
    basestring = str

def parseArguments():
    """
        Parse the command-line options

        :returns: :py:class:`.Clustering`, int, int, int, str -- Clustering
            object, number of trajectory, number of snapshot, number of epoch,
            output path where to write the files
    """
    desc = "Write the information related to the conformation network to file\n"
    parser = argparse.ArgumentParser(description=desc)
    parser.add_argument("epoch", type=str, help="Path to the epoch to search the snapshot")
    parser.add_argument("trajectory", type=int, help="Trajectory number")
    parser.add_argument("snapshot", type=int, help="Snapshot to select (in accepted steps)")
    parser.add_argument("-o", type=str, default=None, help="Output path where to write the files")
    parser.add_argument("--name", type=str, default="pathway.pdb", help="Name of the pdb to write the files")
    parser.add_argument("--top", type=str, default=None, help="Name of the pdb topology for loading non-pdb trajectories")
    args = parser.parse_args()
    return args.trajectory, args.snapshot, args.epoch, args.o, args.name, args.top

def main(trajectory, snapshot, epoch, outputPath, out_filename, topology):
    if outputPath is not None:
        outputPath = os.path.join(outputPath, "")
        if not os.path.exists(outputPath):
            os.makedirs(outputPath)
    else:
        outputPath = ""
    if topology is not None:
        topology_contents = utilities.getTopologyFile(topology)
    else:
        topology_contents = None
    if os.path.exists(outputPath+out_filename):
        # If the specified name exists, append a number to distinguish the files
        name, ext = os.path.splitext(out_filename)
        out_filename = "".join([name, "_%d", ext])
        i = 1
        while os.path.exists(outputPath+out_filename % i):
            i += 1
        out_filename %= i
    pathway = []
    # Strip out trailing backslash if present
    pathPrefix, epoch = os.path.split(epoch.rstrip("/"))
    sys.stderr.write("Creating pathway...\n")
    while True:
        filename = glob.glob(os.path.join(pathPrefix, epoch, "*traj*_%d.*" % trajectory))
        snapshots = utilities.getSnapshots(filename[0], topology=topology)
        if not isinstance(snapshots[0], basestring):
            new_snapshots = []
            for i in range(snapshot+1):
                snapshot = snapshots.slice(i, copy=False)
                PDB = atomset.PDB()
                PDB.initialise(snapshot)
                new_snapshots.append(PDB.get_pdb_string(topology_contents))
            snapshots = new_snapshots
        else:
            snapshots = snapshots[:snapshot+1]
        pathway.insert(0, snapshots)
        if epoch == '0':
            # Once we get to epoch 0, we just need to append the trajectory
            # where the cluster was found and we can break out of the loop
            break
        procMapping = open(os.path.join(pathPrefix, epoch, "processorMapping.txt")).read().rstrip().split(':')
        epoch, trajectory, snapshot = map(int, procMapping[trajectory-1][1:-1].split(','))
        epoch = str(epoch)
    sys.stderr.write("Writing pathway...\n")
    with open(outputPath+out_filename, "a") as f:
        f.write("ENDMDL\n".join(itertools.chain.from_iterable(pathway)))

if __name__ == "__main__":
    traj, num_snapshot, num_epoch, output_path, output_filename, top = parseArguments()
    main(traj, num_snapshot, num_epoch, output_path, output_filename, top)
<|endoftext|>