<commit_before>"""Chrome Browser Cloud Management API calls"""

import csv
import os.path
import sys

import gam
from gam.var import *
from gam import controlflow
from gam import display
from gam import fileutils
from gam import gapi
from gam.gapi.directory import orgunits as gapi_directory_orgunits
from gam import utils

def build():
    return gam.buildGAPIObject('cbcm')

def delete():
    cbcm = build()
    device_id = sys.argv[3]
    gapi.call(cbcm.chromebrowsers(), 'delete', deviceId=device_id,
              customer=GC_Values[GC_CUSTOMER_ID])
    print(f'Deleted browser {device_id}')

def info():
    cbcm = build()
    device_id = sys.argv[3]
    projection = 'BASIC'
    fields = None
    i = 4
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg in ['basic', 'full']:
            projection = myarg.upper()
            i += 1
        elif myarg == 'fields':
            fields = sys.argv[i+1]
            i += 2
        else:
            controlflow.invalid_argument_exit(sys.argv[i], 'gam info browser')
    browser = gapi.call(cbcm.chromebrowsers(), 'get',
                        customer=GC_Values[GC_CUSTOMER_ID],
                        fields=fields, deviceId=device_id,
                        projection=projection)
    display.print_json(browser)

def move():
    cbcm = build()
    body = {'resource_ids': []}
    i = 3
    resource_ids = []
    batch_size = 600
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg == 'ids':
            resource_ids.extend(sys.argv[i + 1].split(','))
            i += 2
        elif myarg == 'query':
            query = sys.argv[i + 1]
            page_message = gapi.got_total_items_msg('Browsers', '...\n')
            browsers = gapi.get_all_pages(cbcm.chromebrowsers(), 'list',
                         'browsers', page_message=page_message,
                         customer=GC_Values[GC_CUSTOMER_ID],
                         query=query, projection='BASIC',
                         fields='browsers(deviceId),nextPageToken')
            ids = [browser['deviceId'] for browser in browsers]
            resource_ids.extend(ids)
            i += 2
        elif myarg == 'file':
            with fileutils.open_file(sys.argv[i+1], strip_utf_bom=True) as filed:
                for row in filed:
                    rid = row.strip()
                    if rid:
                        resource_ids.append(rid)
            i += 2
        elif myarg == 'csvfile':
            drive, fname_column = os.path.splitdrive(sys.argv[i+1])
            if fname_column.find(':') == -1:
                controlflow.system_error_exit(
                    2, 'Expected csvfile FileName:FieldName')
            (filename, column) = fname_column.split(':')
            with fileutils.open_file(drive + filename) as filed:
                input_file = csv.DictReader(filed, restval='')
                if column not in input_file.fieldnames:
                    controlflow.csv_field_error_exit(column,
                                                     input_file.fieldnames)
                for row in input_file:
                    rid = row[column].strip()
                    if rid:
                        resource_ids.append(rid)
            i += 2
        elif myarg in ['ou', 'orgunit', 'org']:
            org_unit = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
            body['org_unit_path'] = org_unit
            i += 2
        elif myarg == 'batchsize':
            batch_size = int(sys.argv[i+1])
            i += 2
        else:
            controlflow.invalid_argument_exit(sys.argv[i],
                                              'gam update browsers')
    if 'org_unit_path' not in body:
        controlflow.missing_argument_exit('ou', 'gam update browsers')
    elif not resource_ids:
        controlflow.missing_argument_exit('query or ids',
                                          'gam update browsers')
    # split moves into max 600 devices per batch
    for chunk in range(0, len(resource_ids), batch_size):
        body['resource_ids'] = resource_ids[chunk:chunk + batch_size]
        print(f' moving {len(body["resource_ids"])} browsers to ' \
                       f'{body["org_unit_path"]}')
        gapi.call(cbcm.chromebrowsers(), 'moveChromeBrowsersToOu',
                  customer=GC_Values[GC_CUSTOMER_ID], body=body)

def print_():
    cbcm = build()
    projection = 'BASIC'
    query = None
    fields = None
    titles = []
    csv_rows = []
    todrive = False
    sort_headers = False
    i = 3
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg == 'query':
            query = sys.argv[i+1]
            i += 2
        elif myarg == 'projection':
            projection = sys.argv[i + 1].upper()
            i += 2
        elif myarg == 'todrive':
            todrive = True
            i += 1
        elif myarg == 'sortheaders':
            sort_headers = True
            i += 1
        elif myarg == 'fields':
            fields = sys.argv[i + 1]
            i += 2
        else:
            controlflow.invalid_argument_exit(sys.argv[i],
                                              'gam print browsers')
    if fields:
        fields = f'browsers({fields}),nextPageToken'
    page_message = gapi.got_total_items_msg('Browsers', '...\n')
    browsers = gapi.get_all_pages(cbcm.chromebrowsers(), 'list',
                         'browsers', page_message=page_message,
                         customer=GC_Values[GC_CUSTOMER_ID],
                         query=query, projection=projection,
                         fields=fields)
    for browser in browsers:
        browser = utils.flatten_json(browser)
        for a_key in browser:
            if a_key not in titles:
                titles.append(a_key)
        csv_rows.append(browser)
    if sort_headers:
        display.sort_csv_titles(['name',], titles)
    display.write_csv_file(csv_rows, titles, 'Browsers', todrive)

def update():
    cbcm = build()
    device_id = sys.argv[3]
    body = {'deviceId': device_id}
    attributes = ['user', 'location', 'notes', 'assetid']
    i = 4
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg in attributes:
            attribute = f'annotated{myarg.capitalize()}'
            body[attribute] = sys.argv[i+1]
            i += 2
        else:
            controlflow.invalid_argument_exit(sys.argv[i],
                                              'gam print browsers')
    result = gapi.call(cbcm.chromebrowsers(), 'update', deviceId=device_id,
                       customer=GC_Values[GC_CUSTOMER_ID], body=body,
                       projection='BASIC', fields="deviceId")
    print(f'Updated browser {result["deviceId"]}')
<commit_msg>Fix browser error messages, item name map in update browser (#1293)<commit_after>"""Chrome Browser Cloud Management API calls"""

import csv
import os.path
import sys

import gam
from gam.var import *
from gam import controlflow
from gam import display
from gam import fileutils
from gam import gapi
from gam.gapi.directory import orgunits as gapi_directory_orgunits
from gam import utils

def build():
    return gam.buildGAPIObject('cbcm')

def delete():
    cbcm = build()
    device_id = sys.argv[3]
    gapi.call(cbcm.chromebrowsers(), 'delete', deviceId=device_id,
              customer=GC_Values[GC_CUSTOMER_ID])
    print(f'Deleted browser {device_id}')

def info():
    cbcm = build()
    device_id = sys.argv[3]
    projection = 'BASIC'
    fields = None
    i = 4
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg in ['basic', 'full']:
            projection = myarg.upper()
            i += 1
        elif myarg == 'fields':
            fields = sys.argv[i+1]
            i += 2
        else:
            controlflow.invalid_argument_exit(sys.argv[i], 'gam info browser')
    browser = gapi.call(cbcm.chromebrowsers(), 'get',
                        customer=GC_Values[GC_CUSTOMER_ID],
                        fields=fields, deviceId=device_id,
                        projection=projection)
    display.print_json(browser)

def move():
    cbcm = build()
    body = {'resource_ids': []}
    i = 3
    resource_ids = []
    batch_size = 600
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg == 'ids':
            resource_ids.extend(sys.argv[i + 1].split(','))
            i += 2
        elif myarg == 'query':
            query = sys.argv[i + 1]
            page_message = gapi.got_total_items_msg('Browsers', '...\n')
            browsers = gapi.get_all_pages(cbcm.chromebrowsers(), 'list',
                         'browsers', page_message=page_message,
                         customer=GC_Values[GC_CUSTOMER_ID],
                         query=query, projection='BASIC',
                         fields='browsers(deviceId),nextPageToken')
            ids = [browser['deviceId'] for browser in browsers]
            resource_ids.extend(ids)
            i += 2
        elif myarg == 'file':
            with fileutils.open_file(sys.argv[i+1], strip_utf_bom=True) as filed:
                for row in filed:
                    rid = row.strip()
                    if rid:
                        resource_ids.append(rid)
            i += 2
        elif myarg == 'csvfile':
            drive, fname_column = os.path.splitdrive(sys.argv[i+1])
            if fname_column.find(':') == -1:
                controlflow.system_error_exit(
                    2, 'Expected csvfile FileName:FieldName')
            (filename, column) = fname_column.split(':')
            with fileutils.open_file(drive + filename) as filed:
                input_file = csv.DictReader(filed, restval='')
                if column not in input_file.fieldnames:
                    controlflow.csv_field_error_exit(column,
                                                     input_file.fieldnames)
                for row in input_file:
                    rid = row[column].strip()
                    if rid:
                        resource_ids.append(rid)
            i += 2
        elif myarg in ['ou', 'orgunit', 'org']:
            org_unit = gapi_directory_orgunits.getOrgUnitItem(sys.argv[i + 1])
            body['org_unit_path'] = org_unit
            i += 2
        elif myarg == 'batchsize':
            batch_size = int(sys.argv[i+1])
            i += 2
        else:
            controlflow.invalid_argument_exit(sys.argv[i],
                                              'gam move browsers')
    if 'org_unit_path' not in body:
        controlflow.missing_argument_exit('ou', 'gam move browsers')
    elif not resource_ids:
        controlflow.missing_argument_exit('query or ids',
                                          'gam move browsers')
    # split moves into max 600 devices per batch
    for chunk in range(0, len(resource_ids), batch_size):
        body['resource_ids'] = resource_ids[chunk:chunk + batch_size]
        print(f' moving {len(body["resource_ids"])} browsers to ' \
                       f'{body["org_unit_path"]}')
        gapi.call(cbcm.chromebrowsers(), 'moveChromeBrowsersToOu',
                  customer=GC_Values[GC_CUSTOMER_ID], body=body)

def print_():
    cbcm = build()
    projection = 'BASIC'
    query = None
    fields = None
    titles = []
    csv_rows = []
    todrive = False
    sort_headers = False
    i = 3
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg == 'query':
            query = sys.argv[i+1]
            i += 2
        elif myarg == 'projection':
            projection = sys.argv[i + 1].upper()
            i += 2
        elif myarg == 'todrive':
            todrive = True
            i += 1
        elif myarg == 'sortheaders':
            sort_headers = True
            i += 1
        elif myarg == 'fields':
            fields = sys.argv[i + 1]
            i += 2
        else:
            controlflow.invalid_argument_exit(sys.argv[i],
                                              'gam print browsers')
    if fields:
        fields = f'browsers({fields}),nextPageToken'
    page_message = gapi.got_total_items_msg('Browsers', '...\n')
    browsers = gapi.get_all_pages(cbcm.chromebrowsers(), 'list',
                         'browsers', page_message=page_message,
                         customer=GC_Values[GC_CUSTOMER_ID],
                         query=query, projection=projection,
                         fields=fields)
    for browser in browsers:
        browser = utils.flatten_json(browser)
        for a_key in browser:
            if a_key not in titles:
                titles.append(a_key)
        csv_rows.append(browser)
    if sort_headers:
        display.sort_csv_titles(['name',], titles)
    display.write_csv_file(csv_rows, titles, 'Browsers', todrive)

attributes = {
    'assetid': 'annotatedAssetId',
    'location': 'annotatedLocation',
    'notes': 'annotatedNotes',
    'user': 'annotatedUser'
    }

def update():
    cbcm = build()
    device_id = sys.argv[3]
    body = {'deviceId': device_id}
    i = 4
    while i < len(sys.argv):
        myarg = sys.argv[i].lower().replace('_', '')
        if myarg in attributes:
            body[attributes[myarg]] = sys.argv[i+1]
            i += 2
        else:
            controlflow.invalid_argument_exit(sys.argv[i],
                                              'gam update browser')
    result = gapi.call(cbcm.chromebrowsers(), 'update', deviceId=device_id,
                       customer=GC_Values[GC_CUSTOMER_ID], body=body,
                       projection='BASIC', fields="deviceId")
    print(f'Updated browser {result["deviceId"]}')
<|endoftext|>


<commit_before># Copyright (c) 2016, German Neuroinformatics Node (G-Node)
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

from . import util

class H5Group(object):

    def __init__(self, parent, name, create=False):
        self._parent = parent
        self.name = name
        self.group = None
        if create or name in self._parent:
            self._create_h5obj()

    def _create_h5obj(self):
        self.group = self._parent.require_group(self.name)

    @classmethod
    def _create_from_h5obj(cls, h5obj):
        parent = h5obj.parent
        name = h5obj.name.split("/")[-1]
        return cls(parent, name)

    def open_group(self, name, create=False):
        """
        Returns a new H5Group with the given name contained in the current group.
        If the current group does not exist in the file, it is automatically
        created.

        :param name: the name of the group
        :param create: creates the child group in the file if it does not exist
        :return: a new H5Group object
        """
        self._create_h5obj()
        return H5Group(self.group, name, create)

    def create_dataset(self, *args, **kwargs):
        """
        Passthrough method. Calls h5py.Group.create_dataset method with given
        arguments but does not return the new object.
        """
        self._create_h5obj()
        self.group.create_dataset(*args, **kwargs)

    def add_by_id(self, id_or_name):
        self._create_h5obj()
        parblock = self.parent.parent.parent
        parcontainer = parblock.open_group(self.name)
        target = parcontainer.get_by_id_or_name(id_or_name)
        self.group[target.get_attr("name")] = target.group

    def has_by_id(self, id_or_name):
        if not self.group:
            return False
        if util.is_uuid(id_or_name):
            for item in self.group:
                if item.attrs["id"] == id_or_name:
                    return True
            else:
                return False
        else:
            return id_or_name in self.group

    def get_data(self, name):
        """
        Returns the data contained in the dataset identified by 'name', or None
        if the a dataset of that name does not exist in the Group.

        :param name: The name of the dataset
        :return: The data contained in the dataset as a numpy array or None
        """
        if name not in self.group:
            return tuple()

        dset = self.group[name]
        # TODO: Error if dset is Group?
        return dset[:]

    def get_by_id_or_name(self, id_or_name):
        if util.is_uuid(id_or_name):
            return self.get_by_id(id_or_name)
        else:
            return self.get_by_name(id_or_name)

    def get_by_name(self, name):
        if self.group and name in self.group:
            return self._create_from_h5obj(self.group[name])
        else:
            raise ValueError("No item with name {} found in {}".format(
                name, self.group.name
            ))

    def get_by_id(self, id_):
        if self.group:
            for item in self.group.values():
                if item.attrs["id"] == id_:
                    return self._create_from_h5obj(item)
        raise ValueError("No item with ID {} found in {}".format(
            id_, self.group.name
        ))

    def get_by_pos(self, pos):
        if not self.group:
            raise ValueError
        return self._create_from_h5obj(list(self.group.values())[pos])

    def delete(self, id_or_name):
        if util.is_uuid(id_or_name):
            name = self.get_by_id_or_name(id_or_name).name
        else:
            name = id_or_name
        try:
            del self.group[name]
        except Exception:
            raise ValueError
        # Delete if empty
        if not len(self.group):
            del self.group
            self.group = None

    def set_attr(self, name, value):
        self._create_h5obj()
        if value is None:
            del self.group.attrs[name]
        else:
            self.group.attrs[name] = value

    def get_attr(self, name):
        return self.group.attrs.get(name)

    @property
    def parent(self):
        return self._create_from_h5obj(self._parent)

    def __contains__(self, item):
        if self.group is None:
            return False
        return item in self.group

    def __len__(self):
        if self.group is None:
            return 0
        else:
            return len(self.group)

    def __str__(self):
        return "<H5Group object: {}>".format(self.group.name)

<commit_msg>Do not delete root containers<commit_after># Copyright (c) 2016, German Neuroinformatics Node (G-Node)
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

from . import util

class H5Group(object):

    def __init__(self, parent, name, create=False):
        self._parent = parent
        self.name = name
        self.group = None
        if create or name in self._parent:
            self._create_h5obj()

    def _create_h5obj(self):
        self.group = self._parent.require_group(self.name)

    @classmethod
    def _create_from_h5obj(cls, h5obj):
        parent = h5obj.parent
        name = h5obj.name.split("/")[-1]
        return cls(parent, name)

    def open_group(self, name, create=False):
        """
        Returns a new H5Group with the given name contained in the current group.
        If the current group does not exist in the file, it is automatically
        created.

        :param name: the name of the group
        :param create: creates the child group in the file if it does not exist
        :return: a new H5Group object
        """
        self._create_h5obj()
        return H5Group(self.group, name, create)

    def create_dataset(self, *args, **kwargs):
        """
        Passthrough method. Calls h5py.Group.create_dataset method with given
        arguments but does not return the new object.
        """
        self._create_h5obj()
        self.group.create_dataset(*args, **kwargs)

    def add_by_id(self, id_or_name):
        self._create_h5obj()
        parblock = self.parent.parent.parent
        parcontainer = parblock.open_group(self.name)
        target = parcontainer.get_by_id_or_name(id_or_name)
        self.group[target.get_attr("name")] = target.group

    def has_by_id(self, id_or_name):
        if not self.group:
            return False
        if util.is_uuid(id_or_name):
            for item in self.group:
                if item.attrs["id"] == id_or_name:
                    return True
            else:
                return False
        else:
            return id_or_name in self.group

    def get_data(self, name):
        """
        Returns the data contained in the dataset identified by 'name', or None
        if the a dataset of that name does not exist in the Group.

        :param name: The name of the dataset
        :return: The data contained in the dataset as a numpy array or None
        """
        if name not in self.group:
            return tuple()

        dset = self.group[name]
        # TODO: Error if dset is Group?
        return dset[:]

    def get_by_id_or_name(self, id_or_name):
        if util.is_uuid(id_or_name):
            return self.get_by_id(id_or_name)
        else:
            return self.get_by_name(id_or_name)

    def get_by_name(self, name):
        if self.group and name in self.group:
            return self._create_from_h5obj(self.group[name])
        else:
            raise ValueError("No item with name {} found in {}".format(
                name, self.group.name
            ))

    def get_by_id(self, id_):
        if self.group:
            for item in self.group.values():
                if item.attrs["id"] == id_:
                    return self._create_from_h5obj(item)
        raise ValueError("No item with ID {} found in {}".format(
            id_, self.group.name
        ))

    def get_by_pos(self, pos):
        if not self.group:
            raise ValueError
        return self._create_from_h5obj(list(self.group.values())[pos])

    def delete(self, id_or_name):
        if util.is_uuid(id_or_name):
            name = self.get_by_id_or_name(id_or_name).name
        else:
            name = id_or_name
        try:
            del self.group[name]
        except Exception:
            raise ValueError
        # Delete if empty and non-root container
        groupdepth = len(self.group.name.split("/")) - 1
        if not len(self.group) and groupdepth > 1:
            del self.group
            self.group = None

    def set_attr(self, name, value):
        self._create_h5obj()
        if value is None:
            del self.group.attrs[name]
        else:
            self.group.attrs[name] = value

    def get_attr(self, name):
        return self.group.attrs.get(name)

    @property
    def parent(self):
        return self._create_from_h5obj(self._parent)

    def __contains__(self, item):
        if self.group is None:
            return False
        return item in self.group

    def __len__(self):
        if self.group is None:
            return 0
        else:
            return len(self.group)

    def __str__(self):
        return "<H5Group object: {}>".format(self.group.name)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Gargantua

Author : Laisky
EMail  : ppcelery@gmail.com
Home   : https://github.com/Laisky/laisky-blog
"""

import logging
from pathlib import Path

import tornado
import motor
import pymongo
from tornado.web import url
from tornado.options import define, options

from gargantua.const import CWD, LISTEN_PORT, LOG_NAME, \
    DB_HOST, DB_PORT, DB_NAME, \
    MAIL_HOST, MAIL_PORT, MAIL_USERNAME, MAIL_PASSWD, FROM_ADDR, TO_ADDRS, MAIL_SUBJECT
from gargantua.utils import setup_log, generate_random_string
from gargantua.views import BaseHandler, PostsHandler, PostPage, \
    PublishHandler, UserHandler, RssHandler, AmendHandler
from gargantua.apis import PostApiHandler

# settings
define('port', default=LISTEN_PORT, type=int)
define('debug', default=False, type=bool)
# database
define('dbname', default=DB_NAME, type=str)
define('dbhost', default=DB_HOST, type=str)
define('dbport', default=DB_PORT, type=int)
# mail
define('mail_passwd', default=MAIL_PASSWD, type=str)

log = logging.getLogger(LOG_NAME)
setup_log()

class PageNotFound(BaseHandler):

    @tornado.gen.coroutine
    def get(self, url=None):
        log.debug('GET PageNotFound for url {}'.format(url))

        if url is None:
            self.render2('404.html', url=url)
            self.finish()
            return

        self.redirect_404()

class Application(tornado.web.Application):

    def __init__(self):
        settings = {
            'static_path': str(Path(CWD, 'static')),
            'static_url_prefix': '/static/',
            'template_path': str(Path(CWD, 'templates')),
            'cookie_secret': generate_random_string(50),
            'login_url': '/login/',
            'xsrf_cookies': True,
            'autoescape': None,
            'debug': options.debug
        }
        handlers = [
            # -------------- handler --------------
            url(r'^/(archives)/$', PostsHandler, name='post:archives'),
            url(r'^/p/(.*)/$', PostPage, name='post:single'),
            url(r'^/publish/$', PublishHandler, name='post:publish'),
            url(r'^/amend/$', AmendHandler, name='post:amend'),
            url(r'^/(login)/$', UserHandler, name='user:login'),
            url(r'^/(search)/$', PostsHandler, name='post:search'),
            url(r'^/(profile)/$', UserHandler, name='user:profile'),
            # ---------------- rss ----------------
            url(r'^/rss.xml$', RssHandler, name='rss'),
            # ---------------- old api ----------------
            url(r'^/(api/posts/.*)/$', PostsHandler, name='api:post'),
            url(r'^/(api/user/.*)/$', UserHandler, name='api:user:login'),
            # ---------------- rest api ----------------
            url(r'^/api/p/([a-zA-Z]+)?/?$', PostApiHandler, name='rest:post'),
            # ---------------- 404 ----------------
            url(r'^/$', PostsHandler, name='root'),
            url(r'^/404.html$', PageNotFound, name='404'),
        ]
        handlers.append(('/(.*)', PageNotFound))
        self.setup_db()
        if not options.debug:
            self.setup_mail_handler()

        super(Application, self).__init__(handlers, **settings)

    def setup_mail_handler(self):
        # set mail handler
        mh = LogMailHandler(mailhost=(MAIL_HOST, MAIL_PORT),
                            fromaddr=FROM_ADDR,
                            toaddrs=TO_ADDRS,
                            credentials=(MAIL_USERNAME, options.mail_passwd),
                            subject=MAIL_SUBJECT)
        mh.setFormatter(LogMailFormatter())
        mh.setLevel(logging.ERROR)
        log.addHandler(mh)

    def setup_db(self):
        log.debug('connect database at {}:{}'
                  .format(options.dbhost, options.dbport))

        self.conn = motor.MotorClient(host=options.dbhost, port=options.dbport)
        self.db = self.conn[options.dbname]
        self.mongo_conn = pymongo.MongoClient(host=options.dbhost, port=options.dbport)
        self.mongo_db = self.mongo_conn[options.dbname]

        # ensure index
        # posts
        # posts_idx = pymongo.IndexModel([('post_name',)], unique=True)
        # self.mongo_db.posts.create_indexes([posts_idx])
        self.mongo_db.posts.ensure_index([('post_name', pymongo.ASCENDING)], unique=True)  # PyMongo2.8
        # users
        # account_idx = pymongo.IndexModel([('account',)], unique=True)
        # username_idx = pymongo.IndexModel([('username',)], unique=True)
        # self.mongo_db.users.create_indexes([account_idx, username_idx])
        self.mongo_db.users.ensure_index([('account', pymongo.ASCENDING)], unique=True)   # PyMongo2.8
        self.mongo_db.users.ensure_index([('username', pymongo.ASCENDING)], unique=True)  # PyMongo2.8
<commit_msg>* fix: 依赖<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""Gargantua

Author : Laisky
EMail  : ppcelery@gmail.com
Home   : https://github.com/Laisky/laisky-blog
"""

import logging
from pathlib import Path

import tornado
import motor
import pymongo
from tornado.web import url
from tornado.options import define, options

from gargantua.const import CWD, LISTEN_PORT, LOG_NAME, \
    DB_HOST, DB_PORT, DB_NAME, \
    MAIL_HOST, MAIL_PORT, MAIL_USERNAME, MAIL_PASSWD, FROM_ADDR, TO_ADDRS, MAIL_SUBJECT
from gargantua.utils import setup_log, generate_random_string
from gargantua.views import BaseHandler, PostsHandler, PostPage, \
    PublishHandler, UserHandler, RssHandler, AmendHandler
from gargantua.apis import PostApiHandler
from gargantua.libs import LogMailHandler, LogMailFormatter

# settings
define('port', default=LISTEN_PORT, type=int)
define('debug', default=False, type=bool)
# database
define('dbname', default=DB_NAME, type=str)
define('dbhost', default=DB_HOST, type=str)
define('dbport', default=DB_PORT, type=int)
# mail
define('mail_passwd', default=MAIL_PASSWD, type=str)

log = logging.getLogger(LOG_NAME)
setup_log()

class PageNotFound(BaseHandler):

    @tornado.gen.coroutine
    def get(self, url=None):
        log.debug('GET PageNotFound for url {}'.format(url))

        if url is None:
            self.render2('404.html', url=url)
            self.finish()
            return

        self.redirect_404()

class Application(tornado.web.Application):

    def __init__(self):
        settings = {
            'static_path': str(Path(CWD, 'static')),
            'static_url_prefix': '/static/',
            'template_path': str(Path(CWD, 'templates')),
            'cookie_secret': generate_random_string(50),
            'login_url': '/login/',
            'xsrf_cookies': True,
            'autoescape': None,
            'debug': options.debug
        }
        handlers = [
            # -------------- handler --------------
            url(r'^/(archives)/$', PostsHandler, name='post:archives'),
            url(r'^/p/(.*)/$', PostPage, name='post:single'),
            url(r'^/publish/$', PublishHandler, name='post:publish'),
            url(r'^/amend/$', AmendHandler, name='post:amend'),
            url(r'^/(login)/$', UserHandler, name='user:login'),
            url(r'^/(search)/$', PostsHandler, name='post:search'),
            url(r'^/(profile)/$', UserHandler, name='user:profile'),
            # ---------------- rss ----------------
            url(r'^/rss.xml$', RssHandler, name='rss'),
            # ---------------- old api ----------------
            url(r'^/(api/posts/.*)/$', PostsHandler, name='api:post'),
            url(r'^/(api/user/.*)/$', UserHandler, name='api:user:login'),
            # ---------------- rest api ----------------
            url(r'^/api/p/([a-zA-Z]+)?/?$', PostApiHandler, name='rest:post'),
            # ---------------- 404 ----------------
            url(r'^/$', PostsHandler, name='root'),
            url(r'^/404.html$', PageNotFound, name='404'),
        ]
        handlers.append(('/(.*)', PageNotFound))
        self.setup_db()
        if not options.debug:
            self.setup_mail_handler()

        super(Application, self).__init__(handlers, **settings)

    def setup_mail_handler(self):
        # set mail handler
        mh = LogMailHandler(mailhost=(MAIL_HOST, MAIL_PORT),
                            fromaddr=FROM_ADDR,
                            toaddrs=TO_ADDRS,
                            credentials=(MAIL_USERNAME, options.mail_passwd),
                            subject=MAIL_SUBJECT)
        mh.setFormatter(LogMailFormatter())
        mh.setLevel(logging.ERROR)
        log.addHandler(mh)

    def setup_db(self):
        log.debug('connect database at {}:{}'
                  .format(options.dbhost, options.dbport))

        self.conn = motor.MotorClient(host=options.dbhost, port=options.dbport)
        self.db = self.conn[options.dbname]
        self.mongo_conn = pymongo.MongoClient(host=options.dbhost, port=options.dbport)
        self.mongo_db = self.mongo_conn[options.dbname]

        # ensure index
        # posts
        # posts_idx = pymongo.IndexModel([('post_name',)], unique=True)
        # self.mongo_db.posts.create_indexes([posts_idx])
        self.mongo_db.posts.ensure_index([('post_name', pymongo.ASCENDING)], unique=True)  # PyMongo2.8
        # users
        # account_idx = pymongo.IndexModel([('account',)], unique=True)
        # username_idx = pymongo.IndexModel([('username',)], unique=True)
        # self.mongo_db.users.create_indexes([account_idx, username_idx])
        self.mongo_db.users.ensure_index([('account', pymongo.ASCENDING)], unique=True)   # PyMongo2.8
        self.mongo_db.users.ensure_index([('username', pymongo.ASCENDING)], unique=True)  # PyMongo2.8
<|endoftext|>


<commit_before>try:
    import cPickle as pickle
except:
    import pickle

from webhelpers.feedgenerator import Rss201rev2Feed
from datetime import datetime
import logging

class Item(object):
    def __init__(self, t, l, a, d):
        self.title = t
        self.link = l
        self.author_name = a
        self.pubdate = datetime.now()
        self.description = d

def notify(header, body, cfg):
    fn = cfg['rss.file']
    pickle_fn = fn + '.pickle'
    try:
        with open(pickle_fn, 'rb') as inp:
            previous = pickle.load(inp)
    except Exception as e:
        logging.error(e)
        previous = []
    previous = previous[:max(10, len(previous))]
    feed = Rss201rev2Feed(
            title=cfg['rss.title'],
            link='',
            description=cfg['rss.description'],
            language=u"en",
        )
    previous.insert(0, Item(header, '', 'author', body))
    for item in previous:
        feed.add_item(title=item.title, description=item.description, link=item.link, author_name=item.author_name,
                       pubdate=item.pubdate)

    with open(cfg['rss.file'], 'wb') as out:
        feed.write(out, 'utf-8')
    with open(pickle_fn, 'wb') as p:
        pickle.dump(previous, p)<commit_msg>fix newlines in rss module<commit_after>try:
    import cPickle as pickle
except:
    import pickle

from webhelpers.feedgenerator import Rss201rev2Feed as Feed
from datetime import datetime
import logging
import cgi

class Item(object):
    def __init__(self, t, l, a, d):
        self.title = t
        self.link = l
        self.author_name = a
        self.pubdate = datetime.now()
        self.description = d.replace('\n', '<br/>')

def notify(header, body, cfg):
    fn = cfg['rss.file']
    pickle_fn = fn + '.pickle'
    try:
        with open(pickle_fn, 'rb') as inp:
            previous = pickle.load(inp)
    except Exception as e:
        logging.error(e)
        previous = []
    previous = previous[:max(10, len(previous))]
    feed = Feed(
            title=cfg['rss.title'],
            link='',
            description=cfg['rss.description'],
            language=u"en",
        )
    previous.insert(0, Item(header, '', 'author', body))
    for item in previous:
        feed.add_item(title=item.title, description=item.description, link=item.link, author_name=item.author_name,
                       pubdate=item.pubdate)

    with open(cfg['rss.file'], 'wb') as out:
        feed.write(out, 'utf-8')
    with open(pickle_fn, 'wb') as p:
        pickle.dump(previous, p)<|endoftext|>


<commit_before>#!/usr/bin/env python2
#
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Author: Erwan Velu <erwan.velu@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from SocketServer import BaseRequestHandler, ThreadingTCPServer
import ConfigParser
import socket
import struct
from health_messages import Health_Message as HM
import health_libs as HL
import health_protocol as HP
import logging
import os
import pprint
import sys
import threading
import time
import yaml

socket_list = {}
lock_socket_list = threading.RLock()
hosts = {}
lock_host = threading.RLock()
hosts_state = {}
results_cpu = {}
serv = 0
NOTHING_RUN = 0
CPU_RUN = 1 << 0
MEMORY_RUN = 1 << 1
STORAGE_RUN = 1 << 2
NETWORK_RUN = 1 << 3

SCHED_FAIR = "Fair"

class SocketHandler(BaseRequestHandler):
    global hosts
    global lock_host
    timeout = 5
    disable_nagle_algorithm = False  # Set TCP_NODELAY socket option

    def handle(self):
        lock_socket_list.acquire()
        socket_list[self.client_address] = self.request
        lock_socket_list.release()

        HP.logger.debug('Got connection from %s' % self.client_address[0])
        while True:
            msg = HP.recv_hm_message(socket_list[self.client_address])
            if not msg:
                continue
            if msg.message != HM.ACK:
                if msg.message == HM.DISCONNECT:
                    HP.logger.debug('Disconnecting from %s' %
                                    self.client_address[0])

                    lock_host.acquire()
                    del hosts[self.client_address]
                    del hosts_state[self.client_address]
                    lock_host.release()

                    socket_list[self.client_address].close()

                    lock_socket_list.acquire()
                    del socket_list[self.client_address]
                    lock_socket_list.release()
                    return
                else:
                    lock_host.acquire()
                    hosts[self.client_address] = msg
                    hosts_state[self.client_address] = NOTHING_RUN
                    lock_host.release()

                    if msg.message == HM.MODULE and msg.action == HM.COMPLETED:
                        if msg.module == HM.CPU:
                            cpu_completed(self.client_address, msg)

def createAndStartServer():
    global serv
    ThreadingTCPServer.allow_reuse_address = True
    serv = ThreadingTCPServer(('', 20000), SocketHandler,
                              bind_and_activate=False)
    l_onoff = 1
    l_linger = 0
    serv.socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
                           struct.pack('ii', l_onoff, l_linger))
    serv.server_bind()
    serv.server_activate()
    HP.logger.info('Starting server')
    serv.serve_forever()        # blocking method

def cpu_completed(host, msg):
    global hosts_state
    global results_cpu
    hosts_state[host] &= ~CPU_RUN
    results_cpu[host] = msg.hw

def get_host_list(item):
    global hosts
    global hosts_state
    selected_hosts = {}
    for host in hosts.keys():
        if hosts_state[host] & item == item:
            selected_hosts[host] = True

    return selected_hosts

def compute_affinity():
    affinity = {}
    for host in hosts.keys():
        hw = hosts[host].hw
        system_id = HL.get_value(hw, "system", "product", "serial")

        if system_id not in affinity.keys():
            affinity[system_id] = [host]
        else:
            # If the system is already known, it means that several
            # virtual machines are sharing the same Hypervisor
            affinity[system_id].append(host)

    return affinity

def get_fair_hosts_list(affinity_hosts_list, nb_hosts):
    hosts_list = []
    while (len(hosts_list) < nb_hosts):
        for hypervisor in affinity_hosts_list.keys():
            hosts_list.append(affinity_hosts_list[hypervisor].pop())
            if (len(hosts_list) == nb_hosts):
                break

    return hosts_list

def get_hosts_list_from_affinity(nb_hosts, affinity):
    affinity_hosts_list = compute_affinity()

    if affinity == SCHED_FAIR:
        return get_fair_hosts_list(affinity_hosts_list, nb_hosts)

def dump_affinity(affinity, selected_host_list, dest_dir):
    HP.logger.info("Using affinity %s on the following mapping" % affinity)
    host_affinity = compute_affinity()
    final_list = {}
    for hypervisor in host_affinity.keys():
        for hostname in selected_host_list:
            if hostname in host_affinity[hypervisor]:
                if hypervisor not in final_list.keys():
                    final_list[hypervisor] = [hostname]
                else:
                    final_list[hypervisor].append(hostname)

    pprint.pprint(final_list)
    pprint.pprint(final_list, stream=open(dest_dir+"/affinity", 'w'))

def start_cpu_bench(nb_hosts, hosts_list, runtime, cores):
    global hosts_state
    msg = HM(HM.MODULE, HM.CPU, HM.START)
    msg.cpu_instances = cores
    msg.running_time = runtime

    for host in hosts_list:
        if nb_hosts == 0:
            break
        if host not in get_host_list(CPU_RUN).keys():
            hosts_state[host] |= CPU_RUN
            nb_hosts = nb_hosts - 1
            lock_socket_list.acquire()
            HP.send_hm_message(socket_list[host], msg)
            lock_socket_list.release()

def disconnect_clients():
    global serv
    msg = HM(HM.DISCONNECT)
    HP.logger.info("Asking %d hosts to disconnect" % len(hosts.keys()))
    for host in hosts.keys():
            lock_socket_list.acquire()
            HP.send_hm_message(socket_list[host], msg)
            lock_socket_list.release()

    while(hosts.keys()):
        time.sleep(1)
        HP.logger.info("Still %d hosts connected" % len(hosts.keys()))

    HP.logger.info("All hosts disconnected")
    serv.shutdown()
    serv.socket.close()

def save_hw(items, name, hwdir):
    'Save hw items for inspection on the server.'
    try:
        filename = os.path.join(hwdir, name + '.hw')
        pprint.pprint(items, stream=open(filename, 'w'))
    except Exception, xcpt:
        HP.logger.error("exception while saving hw file: %s" % str(xcpt))

def compute_results(nb_hosts, affinity, hosts_list):
    config = ConfigParser.ConfigParser()
    config.read('/etc/edeploy.conf')

    def config_get(section, name, default):
        'Secured config getter.'
        try:
            return config.get(section, name)
        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
            return default

    cfg_dir = os.path.normpath(config_get('SERVER', 'HEALTHDIR', '')) + '/'
    dirname = time.strftime("%Y_%m_%d-%Hh%M", time.localtime())
    dest_dir = cfg_dir + 'dahc/%d/' % nb_hosts + dirname

    try:
        if not os.path.isdir(dest_dir):
            os.makedirs(dest_dir)
    except OSError, e:
        HL.fatal_error("Cannot create %s directory (%s)" % (dest_dir, e.errno))

    dump_affinity(affinity, hosts_list, dest_dir)

    for host in results_cpu.keys():
        HP.logger.info("Dumping cpu result from host %s" % str(host))
        filename_and_macs = HL.generate_filename_and_macs(results_cpu[host])
        save_hw(results_cpu[host], filename_and_macs['sysname'], dest_dir)
#        print results_cpu[host]

def get_default_value(job, item, default_value):
    return job.get(item, default_value)

def non_interactive_mode(filename):
    total_runtime = 0
    name = "undefined"

    job = yaml.load(file(filename, 'r'))
    if job['name'] is None:
        HP.logger.error("Missing name parameter in yaml file")
        disconnect_clients()
        return
    else:
        name = job['name']

    if job['required-hosts'] is None:
        HP.logger.error("Missing required-hosts parameter in yaml file")
        disconnect_clients()
        return

    required_hosts = job['required-hosts']
    if required_hosts < 1:
        HP.logger.error("required-hosts shall be greater than 0")
        disconnect_clients()
        return

    runtime = get_default_value(job, 'runtime', 0)

    HP.logger.info("Expecting %d hosts to start job %s" %
                   (required_hosts, name))
    hosts_count = len(hosts.keys())
    previous_hosts_count = hosts_count
    while (int(hosts_count) < int(required_hosts)):
        if (hosts_count != previous_hosts_count):
            HP.logger.info("Still %d hosts to connect" % (int(required_hosts) - int(hosts_count)))
            previous_hosts_count = hosts_count
        hosts_count = len(hosts.keys())
        time.sleep(1)

    HP.logger.info("Starting job %s" % name)
    cpu_job = job['cpu']
    if cpu_job:
            step_hosts = get_default_value(cpu_job, 'step-hosts', 1)
            affinity = get_default_value(cpu_job, 'affinity', SCHED_FAIR)
            required_cpu_hosts = get_default_value(cpu_job, 'required-hosts',
                                                   required_hosts)
            if "-" in str(required_cpu_hosts):
                min_hosts = int(str(required_cpu_hosts).split("-")[0])
                max_hosts = int(str(required_cpu_hosts).split("-")[1])
            else:
                min_hosts = required_cpu_hosts
                max_hosts = min_hosts

            if max_hosts < 1:
                max_hosts = min_hosts
                HP.logger.error("CPU: required-hosts shall be greater than"
                                " 0, defaulting to global required-hosts=%d"
                                % max_hosts)

            if max_hosts > required_hosts:
                HP.logger.error("CPU: The maximum number of hosts to tests"
                                " is greater than the amount of available"
                                " hosts.")
                HP.logger.error("CPU: Canceling Test")
            else:
                for nb_hosts in xrange(min_hosts, max_hosts+1, step_hosts):
                    cpu_runtime = get_default_value(cpu_job, 'runtime',
                                                    runtime)
                    HP.logger.info("CPU: Waiting bench %d / %d (step = %d)"
                                   " to finish on %d hosts : should take"
                                   " %d seconds" % (nb_hosts, max_hosts,
                                                    step_hosts, nb_hosts,
                                                    cpu_runtime))
                    total_runtime += cpu_runtime
                    cores = get_default_value(cpu_job, 'cores', 1)
                    hosts_list = get_hosts_list_from_affinity(nb_hosts, affinity)

                    start_cpu_bench(nb_hosts, hosts_list, cpu_runtime, cores)

                    time.sleep(cpu_runtime)

                    while (get_host_list(CPU_RUN).keys()):
                        time.sleep(1)

                    compute_results(nb_hosts, affinity, hosts_list)

    HP.logger.info("End of job %s" % name)
    disconnect_clients()

if __name__ == '__main__':

    HP.start_log('/var/tmp/health-server.log', logging.DEBUG)

    if len(sys.argv) < 2:
        HP.logger.error("You must provide a yaml file as argument")
        sys.exit(1)

    myThread = threading.Thread(target=createAndStartServer)
    myThread.start()

    non_interactive = threading.Thread(target=non_interactive_mode,
                                       args=tuple([sys.argv[1]]))
    non_interactive.start()
<commit_msg>health-server: Using lower-case scheduling name<commit_after>#!/usr/bin/env python2
#
# Copyright (C) 2014 eNovance SAS <licensing@enovance.com>
#
# Author: Erwan Velu <erwan.velu@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from SocketServer import BaseRequestHandler, ThreadingTCPServer
import ConfigParser
import socket
import struct
from health_messages import Health_Message as HM
import health_libs as HL
import health_protocol as HP
import logging
import os
import pprint
import sys
import threading
import time
import yaml

socket_list = {}
lock_socket_list = threading.RLock()
hosts = {}
lock_host = threading.RLock()
hosts_state = {}
results_cpu = {}
serv = 0
NOTHING_RUN = 0
CPU_RUN = 1 << 0
MEMORY_RUN = 1 << 1
STORAGE_RUN = 1 << 2
NETWORK_RUN = 1 << 3

SCHED_FAIR = "fair"

class SocketHandler(BaseRequestHandler):
    global hosts
    global lock_host
    timeout = 5
    disable_nagle_algorithm = False  # Set TCP_NODELAY socket option

    def handle(self):
        lock_socket_list.acquire()
        socket_list[self.client_address] = self.request
        lock_socket_list.release()

        HP.logger.debug('Got connection from %s' % self.client_address[0])
        while True:
            msg = HP.recv_hm_message(socket_list[self.client_address])
            if not msg:
                continue
            if msg.message != HM.ACK:
                if msg.message == HM.DISCONNECT:
                    HP.logger.debug('Disconnecting from %s' %
                                    self.client_address[0])

                    lock_host.acquire()
                    del hosts[self.client_address]
                    del hosts_state[self.client_address]
                    lock_host.release()

                    socket_list[self.client_address].close()

                    lock_socket_list.acquire()
                    del socket_list[self.client_address]
                    lock_socket_list.release()
                    return
                else:
                    lock_host.acquire()
                    hosts[self.client_address] = msg
                    hosts_state[self.client_address] = NOTHING_RUN
                    lock_host.release()

                    if msg.message == HM.MODULE and msg.action == HM.COMPLETED:
                        if msg.module == HM.CPU:
                            cpu_completed(self.client_address, msg)

def createAndStartServer():
    global serv
    ThreadingTCPServer.allow_reuse_address = True
    serv = ThreadingTCPServer(('', 20000), SocketHandler,
                              bind_and_activate=False)
    l_onoff = 1
    l_linger = 0
    serv.socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
                           struct.pack('ii', l_onoff, l_linger))
    serv.server_bind()
    serv.server_activate()
    HP.logger.info('Starting server')
    serv.serve_forever()        # blocking method

def cpu_completed(host, msg):
    global hosts_state
    global results_cpu
    hosts_state[host] &= ~CPU_RUN
    results_cpu[host] = msg.hw

def get_host_list(item):
    global hosts
    global hosts_state
    selected_hosts = {}
    for host in hosts.keys():
        if hosts_state[host] & item == item:
            selected_hosts[host] = True

    return selected_hosts

def compute_affinity():
    affinity = {}
    for host in hosts.keys():
        hw = hosts[host].hw
        system_id = HL.get_value(hw, "system", "product", "serial")

        if system_id not in affinity.keys():
            affinity[system_id] = [host]
        else:
            # If the system is already known, it means that several
            # virtual machines are sharing the same Hypervisor
            affinity[system_id].append(host)

    return affinity

def get_fair_hosts_list(affinity_hosts_list, nb_hosts):
    hosts_list = []
    while (len(hosts_list) < nb_hosts):
        for hypervisor in affinity_hosts_list.keys():
            hosts_list.append(affinity_hosts_list[hypervisor].pop())
            if (len(hosts_list) == nb_hosts):
                break

    return hosts_list

def get_hosts_list_from_affinity(nb_hosts, affinity):
    affinity_hosts_list = compute_affinity()

    if affinity == SCHED_FAIR:
        return get_fair_hosts_list(affinity_hosts_list, nb_hosts)

def dump_affinity(affinity, selected_host_list, dest_dir):
    HP.logger.info("Using affinity %s on the following mapping" % affinity)
    host_affinity = compute_affinity()
    final_list = {}
    for hypervisor in host_affinity.keys():
        for hostname in selected_host_list:
            if hostname in host_affinity[hypervisor]:
                if hypervisor not in final_list.keys():
                    final_list[hypervisor] = [hostname]
                else:
                    final_list[hypervisor].append(hostname)

    pprint.pprint(final_list)
    pprint.pprint(final_list, stream=open(dest_dir+"/affinity", 'w'))

def start_cpu_bench(nb_hosts, hosts_list, runtime, cores):
    global hosts_state
    msg = HM(HM.MODULE, HM.CPU, HM.START)
    msg.cpu_instances = cores
    msg.running_time = runtime

    for host in hosts_list:
        if nb_hosts == 0:
            break
        if host not in get_host_list(CPU_RUN).keys():
            hosts_state[host] |= CPU_RUN
            nb_hosts = nb_hosts - 1
            lock_socket_list.acquire()
            HP.send_hm_message(socket_list[host], msg)
            lock_socket_list.release()

def disconnect_clients():
    global serv
    msg = HM(HM.DISCONNECT)
    HP.logger.info("Asking %d hosts to disconnect" % len(hosts.keys()))
    for host in hosts.keys():
            lock_socket_list.acquire()
            HP.send_hm_message(socket_list[host], msg)
            lock_socket_list.release()

    while(hosts.keys()):
        time.sleep(1)
        HP.logger.info("Still %d hosts connected" % len(hosts.keys()))

    HP.logger.info("All hosts disconnected")
    serv.shutdown()
    serv.socket.close()

def save_hw(items, name, hwdir):
    'Save hw items for inspection on the server.'
    try:
        filename = os.path.join(hwdir, name + '.hw')
        pprint.pprint(items, stream=open(filename, 'w'))
    except Exception, xcpt:
        HP.logger.error("exception while saving hw file: %s" % str(xcpt))

def compute_results(nb_hosts, affinity, hosts_list):
    config = ConfigParser.ConfigParser()
    config.read('/etc/edeploy.conf')

    def config_get(section, name, default):
        'Secured config getter.'
        try:
            return config.get(section, name)
        except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
            return default

    cfg_dir = os.path.normpath(config_get('SERVER', 'HEALTHDIR', '')) + '/'
    dirname = time.strftime("%Y_%m_%d-%Hh%M", time.localtime())
    dest_dir = cfg_dir + 'dahc/%d/' % nb_hosts + dirname

    try:
        if not os.path.isdir(dest_dir):
            os.makedirs(dest_dir)
    except OSError, e:
        HL.fatal_error("Cannot create %s directory (%s)" % (dest_dir, e.errno))

    dump_affinity(affinity, hosts_list, dest_dir)

    for host in results_cpu.keys():
        HP.logger.info("Dumping cpu result from host %s" % str(host))
        filename_and_macs = HL.generate_filename_and_macs(results_cpu[host])
        save_hw(results_cpu[host], filename_and_macs['sysname'], dest_dir)
#        print results_cpu[host]

def get_default_value(job, item, default_value):
    return job.get(item, default_value)

def non_interactive_mode(filename):
    total_runtime = 0
    name = "undefined"

    job = yaml.load(file(filename, 'r'))
    if job['name'] is None:
        HP.logger.error("Missing name parameter in yaml file")
        disconnect_clients()
        return
    else:
        name = job['name']

    if job['required-hosts'] is None:
        HP.logger.error("Missing required-hosts parameter in yaml file")
        disconnect_clients()
        return

    required_hosts = job['required-hosts']
    if required_hosts < 1:
        HP.logger.error("required-hosts shall be greater than 0")
        disconnect_clients()
        return

    runtime = get_default_value(job, 'runtime', 0)

    HP.logger.info("Expecting %d hosts to start job %s" %
                   (required_hosts, name))
    hosts_count = len(hosts.keys())
    previous_hosts_count = hosts_count
    while (int(hosts_count) < int(required_hosts)):
        if (hosts_count != previous_hosts_count):
            HP.logger.info("Still %d hosts to connect" % (int(required_hosts) - int(hosts_count)))
            previous_hosts_count = hosts_count
        hosts_count = len(hosts.keys())
        time.sleep(1)

    HP.logger.info("Starting job %s" % name)
    cpu_job = job['cpu']
    if cpu_job:
            step_hosts = get_default_value(cpu_job, 'step-hosts', 1)
            affinity = get_default_value(cpu_job, 'affinity', SCHED_FAIR)
            required_cpu_hosts = get_default_value(cpu_job, 'required-hosts',
                                                   required_hosts)
            if "-" in str(required_cpu_hosts):
                min_hosts = int(str(required_cpu_hosts).split("-")[0])
                max_hosts = int(str(required_cpu_hosts).split("-")[1])
            else:
                min_hosts = required_cpu_hosts
                max_hosts = min_hosts

            if max_hosts < 1:
                max_hosts = min_hosts
                HP.logger.error("CPU: required-hosts shall be greater than"
                                " 0, defaulting to global required-hosts=%d"
                                % max_hosts)

            if max_hosts > required_hosts:
                HP.logger.error("CPU: The maximum number of hosts to tests"
                                " is greater than the amount of available"
                                " hosts.")
                HP.logger.error("CPU: Canceling Test")
            else:
                for nb_hosts in xrange(min_hosts, max_hosts+1, step_hosts):
                    cpu_runtime = get_default_value(cpu_job, 'runtime',
                                                    runtime)
                    HP.logger.info("CPU: Waiting bench %d / %d (step = %d)"
                                   " to finish on %d hosts : should take"
                                   " %d seconds" % (nb_hosts, max_hosts,
                                                    step_hosts, nb_hosts,
                                                    cpu_runtime))
                    total_runtime += cpu_runtime
                    cores = get_default_value(cpu_job, 'cores', 1)
                    hosts_list = get_hosts_list_from_affinity(nb_hosts, affinity)

                    start_cpu_bench(nb_hosts, hosts_list, cpu_runtime, cores)

                    time.sleep(cpu_runtime)

                    while (get_host_list(CPU_RUN).keys()):
                        time.sleep(1)

                    compute_results(nb_hosts, affinity, hosts_list)

    HP.logger.info("End of job %s" % name)
    disconnect_clients()

if __name__ == '__main__':

    HP.start_log('/var/tmp/health-server.log', logging.DEBUG)

    if len(sys.argv) < 2:
        HP.logger.error("You must provide a yaml file as argument")
        sys.exit(1)

    myThread = threading.Thread(target=createAndStartServer)
    myThread.start()

    non_interactive = threading.Thread(target=non_interactive_mode,
                                       args=tuple([sys.argv[1]]))
    non_interactive.start()
<|endoftext|>


<commit_before># Geometric functionality

import math

try:
    # Faster C versions.
    import cGeo
    isqrt = inverse_sqrt = cGeo.fast_inverse_sqrt
    angle = cGeo.angle
    distance = cGeo.distance
    coordinates = cGeo.coordinates

except ImportError:
    def inverse_sqrt(x):
        return 1.0 / math.sqrt(x)

    isqrt = inverse_sqrt

    def angle(x0, y0, x1, y1):
        return math.degrees( math.atan2(y1-y0, x1-x0) )

    def distance(x0, y0, x1, y1):
        return math.sqrt(math.pow(x1-x0, 2) + math.pow(y1-y0, 2))

    def coordinates(x0, y0, distance, angle):
        x1 = x0 + math.cos(math.radians(angle)) * distance
        y1 = y0 + math.sin(math.radians(angle)) * distance
        return x1, y1

def reflect(x0, y0, x1, y1, d=1.0, a=180):
    d *= distance(x0, y0, x1, y1)
    a += angle(x0, y0, x1, y1)
    x, y = coordinates(x0, y0, d, a)
    return x, y

<commit_msg>Added atkinsondither c-extension import.<commit_after># Geometric functionality

import math

try:
    # Faster C versions.
    import cGeo
    isqrt = inverse_sqrt = cGeo.fast_inverse_sqrt
    angle = cGeo.angle
    distance = cGeo.distance
    coordinates = cGeo.coordinates

except ImportError:
    def inverse_sqrt(x):
        return 1.0 / math.sqrt(x)

    isqrt = inverse_sqrt

    def angle(x0, y0, x1, y1):
        return math.degrees( math.atan2(y1-y0, x1-x0) )

    def distance(x0, y0, x1, y1):
        return math.sqrt(math.pow(x1-x0, 2) + math.pow(y1-y0, 2))

    def coordinates(x0, y0, distance, angle):
        x1 = x0 + math.cos(math.radians(angle)) * distance
        y1 = y0 + math.sin(math.radians(angle)) * distance
        return x1, y1

try:
    import atkinsondither
    atkinson = atkinsondither.atkinson

except ImportError:
    def atkinson(*args):
        print "You lost."

def reflect(x0, y0, x1, y1, d=1.0, a=180):
    d *= distance(x0, y0, x1, y1)
    a += angle(x0, y0, x1, y1)
    x, y = coordinates(x0, y0, d, a)
    return x, y

<|endoftext|>


<commit_before>#!/usr/bin/env python3
import rospy
import sys

import cozmo
from cozmo.util import degrees
import matplotlib.pyplot as plt

from visualization_msgs.msg import Marker

# Finds the corner coordinates of a 4 sided surface
# Assumes Cozmo starts perpendicular to an edge
# Cozmo will visit each edge once
def perpendicular_detector(robot: cozmo.robot.Robot):
    x_min = sys.maxsize
    x_max = -sys.maxsize - 1
    y_min = sys.maxsize
    y_max = -sys.maxsize - 1
    x = []
    y = []

    corners = []
    robot.set_lift_height(1).wait_for_completed()

    for angle in [180, 90, 180, 180]:
        while not robot.is_cliff_detected:
            robot.drive_wheels(100, 100, duration=0.25)
        x_curr = robot.pose.position.x
        y_curr = robot.pose.position.y
        x.append(x_curr)
        y.append(y_curr)

        x_min = min(x_curr, x_min)
        x_max = max(x_curr, x_max)
        y_min = min(y_curr, y_min)
        y_max = max(y_curr, y_max)
        robot.drive_wheels(-100, -100, duration=1.5)
        robot.turn_in_place(degrees(angle)).wait_for_completed()

    plt.figure(0)
    plt.scatter(x, y)
    plt.plot(x[0:2], y[0:2], 'r')
    plt.plot(x[2:4], y[2:4], 'r')

    plt.scatter([x_min, x_min, x_max, x_max], [y_min, y_max, y_min, y_max])
    plt.plot([x_min, x_max], [y_min, y_min], 'b')
    plt.plot([x_min, x_max], [y_max, y_max], 'b')
    plt.plot([x_min, x_min], [y_min, y_max], 'b')
    plt.plot([x_max, x_max], [y_min, y_max], 'b')

    plt.savefig('edge.png')
    plt.show()

# Finds the corner coordinates of a 4 sided surface
# Cozmo will continuously find coordinates of the edges of the surface
# until new coordinates are no longer expanding the known dimensions
def generic_detector(robot: cozmo.robot.Robot):
    x_min = sys.maxsize
    x_max = -sys.maxsize - 1
    y_min = sys.maxsize
    y_max = -sys.maxsize - 1

    x = []
    y = []

    corners = []
    robot.set_lift_height(1).wait_for_completed()
    done = False

    while not done:
        while not robot.is_cliff_detected:
            robot.drive_wheels(100, 100, duration=0.25)
        x_curr = robot.pose.position.x
        y_curr = robot.pose.position.y
        x.append(x_curr)
        y.append(y_curr)
        robot.drive_wheels(-100, -100, duration=1.5)

        robot.turn_in_place(degrees(91)).wait_for_completed()

        x_min = min(x_curr, x_min)
        x_max = max(x_curr, x_max)
        y_min = min(y_curr, y_min)
        y_max = max(y_curr, y_max)

        if x_min != x_curr and x_max != x_curr and y_min != y_curr and y_max != y_curr:
            done = True

    plt.figure(0)
    plt.scatter(x, y)
    plt.plot(x, y, 'r')

    plt.scatter([x_min, x_min, x_max, x_max], [y_min, y_max, y_min, y_max])
    plt.plot([x_min, x_max], [y_min, y_min], 'b')
    plt.plot([x_min, x_max], [y_max, y_max], 'b')
    plt.plot([x_min, x_min], [y_min, y_max], 'b')
    plt.plot([x_max, x_max], [y_min, y_max], 'b')
    plt.savefig('generic_edge_detector')

    plt.show()

# Publish the surface found based on the 4 corners
def publish_plane(pub, dimensions):
    plane_marker = Marker()
    plane_marker.header.frame_id = "base_link"
    plane_marker.type = Marker.CUBE

    plane_marker.pose.position.x = 0
    plane_marker.pose.position.y = 0
    plane_marker.pose.position.z = 0

    plane_marker.pose.orientation.x = 0
    plane_marker.pose.orientation.y = 0
    plane_marker.pose.orientation.z = 0
    plane_marker.pose.orientation.w = 1

    plane_marker.scale.x = dimensions[0] / 1000.0
    plane_marker.scale.y = dimensions[1] / 1000.0
    plane_marker.scale.z = 1 / 1000.0

    plane_marker.color.r = 0.0
    plane_marker.color.g = 0.5
    plane_marker.color.b = 0.5
    plane_marker.color.a = 1.0

    pub.publish(plane_marker)

if __name__ == '__main__':
    rospy.init_node("edge_detection")
    try:
        cozmo.run_program(perpendicular_detector)
    except rospy.ROSInterruptException:
        pass
<commit_msg>Revised structure and working now with threshold<commit_after>#!/usr/bin/env python3
import rospy
import sys

import cozmo
from cozmo.util import degrees
import matplotlib.pyplot as plt

from visualization_msgs.msg import Marker

class CornerDetector(object):
    """
    A corner detector class that uses cozmo's sensor to map the dimensions of a
    rectangular surface
    """
    def __init__(self, cozmo, threshold):
        """
        Parameters
        ----------
        cozmo : cozmo.robot
            the cozmo SDK robot handle
        threshold : float
            the distance away from an edge to be within bounds, in mm
        """
        self.cozmo = cozmo
        self.x_min = sys.maxsize
        self.x_max = -sys.maxsize - 1
        self.y_min = sys.maxsize
        self.y_max = -sys.maxsize - 1
        self.threshold = threshold

    def find_corners(self):
        """
        Finds the corner coordinates of a 4 sided surface with the constraint
        that cozmo needs to start perpendicular to an edge
        """
        x = []
        y = []

        corners = []
        self.cozmo.set_lift_height(1).wait_for_completed()

        # The order in which cozmo will turn and find edges
        for angle in [180, 90, 180, 180]:
            while not self.cozmo.is_cliff_detected:
                self.cozmo.drive_wheels(100, 100, duration=0.25)
            x_curr = self.cozmo.pose.position.x
            y_curr = self.cozmo.pose.position.y
            x.append(x_curr)
            y.append(y_curr)

            self.x_min = min(x_curr, self.x_min)
            self.x_max = max(x_curr, self.x_max)
            self.y_min = min(y_curr, self.y_min)
            self.y_max = max(y_curr, self.y_max)
            self.cozmo.drive_wheels(-100, -100, duration=1.5)
            self.cozmo.turn_in_place(degrees(angle)).wait_for_completed()

    def within_bounds(self, x, y):
        """
        Given x, y coordinates (in mm), returns if the coordinates are within bounds
        """
        return self.x_min + threshold <= x <= self.x_max - threshold and \
               self.y_min + threshold <= y <= self.y_max - threshold

    def publish_plane(self, pub, x, y, z, color=(0, 0.5, 0.5)):
        """
        Publish the surface found based on x and y dimensions

        Parameters
        ----------
        pub : ros publisher
        x : float
            The x dimension of the surface
        y : float
            The y dimension of the surface
        z : float
            The height of the surface
        All position and scale units are in mm
        """
        plane_marker = Marker()
        plane_marker.header.frame_id = "base_link"
        plane_marker.type = Marker.CUBE

        plane_marker.pose.position.x = 0
        plane_marker.pose.position.y = 0
        plane_marker.pose.position.z = z / 1000

        plane_marker.pose.orientation.x = 0
        plane_marker.pose.orientation.y = 0
        plane_marker.pose.orientation.z = 0
        plane_marker.pose.orientation.w = 1

        plane_marker.scale.x = x / 1000.0
        plane_marker.scale.y = y / 1000.0
        plane_marker.scale.z = 1 / 1000.0

        plane_marker.color.r = color[0]
        plane_marker.color.g = color[1]
        plane_marker.color.b = color[2]
        plane_marker.color.a = 1

        pub.publish(plane_marker)

def cozmo_run(robot: cozmo.robot):
    detector = CornerDetector(robot, 10)
    plane_publisher = rospy.Publisher('plane_marker', Marker, queue_size=10)
    boundary_publisher = rospy.Publisher('boundary_marker', Marker, queue_size=10)

    detector.find_corners()
    x = detector.x_max - detector.x_min
    y = detector.y_max - detector.y_min
    while not rospy.is_shutdown():
        detector.publish_plane(plane_publisher, x, y, 0, color=(1, 0.5, 0))
        detector.publish_plane(boundary_publisher, x - 2 * detector.threshold, y - 2 * detector.threshold, 2)

if __name__ == '__main__':
    rospy.init_node("edge_detection")
    try:
        cozmo.run_program(cozmo_run)
    except rospy.ROSInterruptException:
        pass
<|endoftext|>


<commit_before>import GPflow
import tensorflow as tf
import os
import numpy as np
from matplotlib import pyplot as plt
import cProfile

def outputGraph( model, dirName, fileName ):
    if not( os.path.isdir(dirName) ):
        os.mkdir(dirName)
    fullFileName = os.path.join( dirName, fileName )
    if os.path.isfile( fullFileName ):
        os.remove( fullFileName )
    tf.train.write_graph(model._session.graph_def, dirName+'/', fileName, as_text=False)

# build a very simple data set:
def getData():
    rng = np.random.RandomState( 1 )
    N = 30
    X = rng.rand(N,1)
    Y = np.sin(12*X) + 0.66*np.cos(25*X) + rng.randn(N,1)*0.1 + 3
    return X,Y

def plotData(X,Y):
    plt.figure()
    plt.plot(X, Y, 'kx', mew=2)

def getRegressionModel(X,Y):
    #build the GPR object
    k = GPflow.kernels.Matern52(1)
    meanf = GPflow.mean_functions.Linear(1,0)
    m = GPflow.gpr.GPR(X, Y, k, meanf)
    m.likelihood.variance = 0.01
    print "Here are the parameters before optimization"
    m
    return m

def optimizeModel(m):
    m.optimize()
    print "Here are the parameters after optimization"
    m

def plotOptimizationResult(X,Y,m):
    #plot!
    xx = np.linspace(-0.1, 1.1, 100)[:,None]
    mean, var = m.predict_y(xx)
    plt.figure()
    plt.plot(X, Y, 'kx', mew=2)
    plt.plot(xx, mean, 'b', lw=2)
    plt.plot(xx, mean + 2*np.sqrt(var), 'b--', xx, mean - 2*np.sqrt(var), 'b--', lw=1.2)

def setModelPriors( m ):
    #we'll choose rather arbitrary priors.
    m.kern.lengthscales.prior = GPflow.priors.Gamma(1., 1.)
    m.kern.variance.prior = GPflow.priors.Gamma(1., 1.)
    m.likelihood.variance.prior = GPflow.priors.Gamma(1., 1.)
    m.mean_function.A.prior = GPflow.priors.Gaussian(0., 10.)
    m.mean_function.b.prior = GPflow.priors.Gaussian(0., 10.)
    print "model with priors ", m

def getSamples( m ):
    samples = m.sample(100, epsilon = 0.1)
    return samples

def plotSamples( X, Y, m, samples ):
    xx = np.linspace(-0.1, 1.1, 100)[:,None]
    plt.figure()
    plt.plot(samples)

    f, axs = plt.subplots(1,3, figsize=(12,4), tight_layout=True)
    axs[0].plot(samples[:,0], samples[:,1], 'k.', alpha = 0.15)
    axs[0].set_xlabel('noise_variance')
    axs[0].set_ylabel('signal_variance')
    axs[1].plot(samples[:,0], samples[:,2], 'k.', alpha = 0.15)
    axs[1].set_xlabel('noise_variance')
    axs[1].set_ylabel('lengthscale')
    axs[2].plot(samples[:,2], samples[:,1], 'k.', alpha = 0.1)
    axs[2].set_xlabel('lengthscale')
    axs[2].set_ylabel('signal_variance')

    #an attempt to plot the function posterior
    #Note that we should really sample the function values here, instead of just using the mean.
    #We are under-representing the uncertainty here.
    # TODO: get full_covariance of the predictions (predict_f only?)

    plt.figure()

    for s in samples:
        m.set_state(s)
        mean, _ = m.predict_y(xx)
        plt.plot(xx, mean, 'b', lw=2, alpha = 0.05)

    plt.plot(X, Y, 'kx', mew=2)

def showAllPlots():
    plt.show()

def runExperiments(plotting=True,outputGraphs=False):
    X,Y = getData()
    if plotting:
        plotData(X,Y)
    m = getRegressionModel(X,Y)
    if outputGraphs:
        modelDir = 'models'
        outputGraph( m, modelDir, 'pointHypers' )
    optimizeModel(m)
    if plotting:
        plotOptimizationResult(X,Y,m)
    setModelPriors( m )
    if outputGraphs:
        outputGraph( m, modelDir, 'bayesHypers' )
    samples = getSamples( m )
    if plotting:
        plotSamples( X, Y, m,  samples )
        showAllPlots()

if __name__ == '__main__':
    runExperiments()
    #cProfile.run( 'runExperiments( plotting=False )' )
<commit_msg>need to compile the graph.<commit_after>import GPflow
import tensorflow as tf
import os
import numpy as np
from matplotlib import pyplot as plt
import cProfile

def outputGraph( model, dirName, fileName ):
    model._compile()
    if not( os.path.isdir(dirName) ):
        os.mkdir(dirName)
    fullFileName = os.path.join( dirName, fileName )
    if os.path.isfile( fullFileName ):
        os.remove( fullFileName )
    tf.train.write_graph(model._session.graph_def, dirName+'/', fileName, as_text=False)

# build a very simple data set:
def getData():
    rng = np.random.RandomState( 1 )
    N = 30
    X = rng.rand(N,1)
    Y = np.sin(12*X) + 0.66*np.cos(25*X) + rng.randn(N,1)*0.1 + 3
    return X,Y

def plotData(X,Y):
    plt.figure()
    plt.plot(X, Y, 'kx', mew=2)

def getRegressionModel(X,Y):
    #build the GPR object
    k = GPflow.kernels.Matern52(1)
    meanf = GPflow.mean_functions.Linear(1,0)
    m = GPflow.gpr.GPR(X, Y, k, meanf)
    m.likelihood.variance = 0.01
    print "Here are the parameters before optimization"
    m
    return m

def optimizeModel(m):
    m.optimize()
    print "Here are the parameters after optimization"
    m

def plotOptimizationResult(X,Y,m):
    #plot!
    xx = np.linspace(-0.1, 1.1, 100)[:,None]
    mean, var = m.predict_y(xx)
    plt.figure()
    plt.plot(X, Y, 'kx', mew=2)
    plt.plot(xx, mean, 'b', lw=2)
    plt.plot(xx, mean + 2*np.sqrt(var), 'b--', xx, mean - 2*np.sqrt(var), 'b--', lw=1.2)

def setModelPriors( m ):
    #we'll choose rather arbitrary priors.
    m.kern.lengthscales.prior = GPflow.priors.Gamma(1., 1.)
    m.kern.variance.prior = GPflow.priors.Gamma(1., 1.)
    m.likelihood.variance.prior = GPflow.priors.Gamma(1., 1.)
    m.mean_function.A.prior = GPflow.priors.Gaussian(0., 10.)
    m.mean_function.b.prior = GPflow.priors.Gaussian(0., 10.)
    print "model with priors ", m

def getSamples( m ):
    samples = m.sample(100, epsilon = 0.1)
    return samples

def plotSamples( X, Y, m, samples ):
    xx = np.linspace(-0.1, 1.1, 100)[:,None]
    plt.figure()
    plt.plot(samples)

    f, axs = plt.subplots(1,3, figsize=(12,4), tight_layout=True)
    axs[0].plot(samples[:,0], samples[:,1], 'k.', alpha = 0.15)
    axs[0].set_xlabel('noise_variance')
    axs[0].set_ylabel('signal_variance')
    axs[1].plot(samples[:,0], samples[:,2], 'k.', alpha = 0.15)
    axs[1].set_xlabel('noise_variance')
    axs[1].set_ylabel('lengthscale')
    axs[2].plot(samples[:,2], samples[:,1], 'k.', alpha = 0.1)
    axs[2].set_xlabel('lengthscale')
    axs[2].set_ylabel('signal_variance')

    #an attempt to plot the function posterior
    #Note that we should really sample the function values here, instead of just using the mean.
    #We are under-representing the uncertainty here.
    # TODO: get full_covariance of the predictions (predict_f only?)

    plt.figure()

    for s in samples:
        m.set_state(s)
        mean, _ = m.predict_y(xx)
        plt.plot(xx, mean, 'b', lw=2, alpha = 0.05)

    plt.plot(X, Y, 'kx', mew=2)

def showAllPlots():
    plt.show()

def runExperiments(plotting=True,outputGraphs=False):
    X,Y = getData()
    if plotting:
        plotData(X,Y)
    m = getRegressionModel(X,Y)
    if outputGraphs:
        modelDir = 'models'
        outputGraph( m, modelDir, 'pointHypers' )
    optimizeModel(m)
    if plotting:
        plotOptimizationResult(X,Y,m)
    setModelPriors( m )
    if outputGraphs:
        outputGraph( m, modelDir, 'bayesHypers' )
    samples = getSamples( m )
    if plotting:
        plotSamples( X, Y, m,  samples )
        showAllPlots()

if __name__ == '__main__':
    runExperiments()
    #cProfile.run( 'runExperiments( plotting=False )' )
<|endoftext|>


<commit_before>#SETUP
from tkinter import *
wins = ["000102","101112","202122","001020","011121","021222","001122","021120"]
n = "x"
won = False
window = Tk()
window.resizable(0,0)
fr = Frame(window)
fr.pack()
w = Label(fr, text="Noughts And Crosses")
btn = []
num = 1
#important functions
def centre(win):
    win.update_idletasks()
    width = win.winfo_width()
    height = win.winfo_height()
    x = (win.winfo_screenwidth() // 2) - (width // 2)
    y = (win.winfo_screenheight() // 2) - (height*2)
    win.geometry('{}x{}+{}+{}'.format(width, height, x, y))
def shift(seq):
    l = seq[1:]
    l.append(seq[0])
    return l
def pg():
    row_frame = Frame(fr)
    row_frame.pack(side="top")
    playag = bt(row_frame, text="Play Again")
    playag.config(command=playag.play)
    playag.pack()
def playagain():
    global fr
    global won
    global n
    global w
    global btn
    global num
    num = 1
    window.geometry("200x200")
    fr.destroy()
    won = False
    n = "x"
    fr = Frame(window)
    fr.pack()
    BoardValue = ["-","-","-","-","-","-","-","-","-"]
    w = Label(fr, text="Noughts And Crosses")
    w.pack()
    btn = []
    for i, b in enumerate(BoardValue):
        if i%3 == 0:
            row_frame = Frame(fr)
            row_frame.pack(side="top")
        btn.append(bt(row_frame, text=b, relief=GROOVE, width=4,height=2))
        btn[i].config(command=btn[i].x)
        btn[i].nam(str(i//3)+str(i%3))
        btn[i].pack(side="left")

#AI CLASS
class AI:
    def checkNear(self,let):
        global btn
        global wins
        for j in wins:
            n1 = (int(j[0])*3) + int(j[1])
            n2 = (int(j[2])*3) + int(j[3])
            n3 = (int(j[4])*3) + int(j[5])
            btns = [btn[n1],btn[n2],btn[n3]]
            for i in range(3):
                if (btns[0].gt() == btns[1].gt()) and (btns[2].gt() == "-") and (btns[1].gt() == let):
                    return btns[2].nams
                    break
                btns = shift(btns)

    def firstmove(self):
        global btn
        if btn[4].cget('state') == 'normal':
            return "11"
        else:
            return "00"

    def order(self):
        global btn
        for x in [1,3,5,7,0,2,6,8,4]:
                if btn[x].cget('state') == 'normal':
                    return str(x//3)+str(x%3)

    def diag(self):
        global btn
        ps = {
            '00': [[1,3],[2,3],[1,6]],
            '02': [[1,5],[2,5],[2,8]],
            '20': [[3,7],[3,8],[0,7]],
            '22': [[5,7],[2,7],[5,6]]
            }
        for a in ps:
            for b in ps[a]:
                if btn[b[0]].gt() == btn[b[1]].gt() and btn[b[0]].gt() == "X":
                    if btn[(int(a[0])*3) + int(a[1])].cget('state') == 'normal':
                        return a

    def miderr(self):
        global btn
        if btn[4].gt() == btn[8].gt() and btn[4].gt() == "X":
            return '02'

    def move(self,n):
        global btn
        if n == 1:
            return self.firstmove()
        else:
            ms = []
            co = self.checkNear("O")
            ms.append(co)
            cx = self.checkNear("X")
            ms.append(cx)
            if n == 2:
                m = self.miderr()
                ms.append(m)
            if n <= 3:
                d = self.diag()
                ms.append(d)
            o = self.order()
            ms.append(o)
            return next(x for x in ms if x != None)
#END AI CLASS

ai = AI() #defining game AI for 1-player game
#switch functions
def switch1():
    global ai
    global n
    global num
    global btn
    if n == "x":
        for i in btn:
            i.config(command=i.o)
        n = "o"
        try:
            mov = ai.move(num)
            num += 1
            but = (int(mov[0])*3) + int(mov[1])
            btn[but].invoke()
        except:
            None
    else:
        for i in btn:
            i.config(command=i.x)
        n = "x"
def switch2():
    global n
    if n == "x":
        for i in btn:
            i.config(command=i.o)
        n = "o"
    else:
        for i in btn:
            i.config(command=i.x)
        n = "x"
switch = switch2
#more functions
def checkEqual(iterator):
    try:
        return all('disabled' == rest.cget('state') for rest in iterator)
    except StopIteration:
        return True
def chw():
    global btn
    global won
    global w
    for j in wins:
        n1 = (int(j[0])*3) + int(j[1])
        n2 = (int(j[2])*3) + int(j[3])
        n3 = (int(j[4])*3) + int(j[5])
        btns = [btn[n1],btn[n2],btn[n3]]
        if (btns[0].gt() == btns[1].gt()) and (btns[2].gt() == btns[1].gt()) and (btns[0].gt() != "-") and won == False:
            for a in btns:
                a.config(bg="red", disabledforeground="white")
            win = "Noughts and Crosses - %s wins!" %(btns[0].gt())
            won = True
            w.config(text=win)
            for x in btn:
                x.config(state="disabled")
            pg()
    if checkEqual(btn) and not(won):
        w.config(text="Noughts and Crosses - It's a draw!")
        pg()
#modified button class
class bt(Button):
    def p1(self):
        global switch
        switch = switch1
        playagain()
    def p2(self):
        global switch
        switch = switch2
        playagain()
    def play(self):
        playagain()
    def gt(self):
        return self.cget('text')
    def nam(self,nam):
        self.nams = nam
    def x(self):
        self.config(text="X",state="disabled")
        chw()
        switch()
    def o(self):
        self.config(text="O",state="disabled")
        switch()
        chw()
#setup window
BoardValue = ["-","-","-","-","-","-","-","-","-"]
window.title("Noughts And Crosses")
window.geometry("200x100")
centre(window)
w.pack()
#setup gamemode choice
play1 = bt(fr,text="1-player")
play2 = bt(fr,text="2-player")
play1.config(command=play1.p1)
play2.config(command=play1.p2)
play1.pack(fill=X)
play2.pack(fill=X)
window.mainloop()
<commit_msg>Icon update to pyw code<commit_after>#SETUP
from tkinter import *
wins = ["000102","101112","202122","001020","011121","021222","001122","021120"]
n = "x"
won = False
window = Tk()
window.resizable(0,0)
fr = Frame(window)
fr.pack()
w = Label(fr, text="Noughts And Crosses")
btn = []
num = 1
#important functions
def centre(win):
    win.update_idletasks()
    width = win.winfo_width()
    height = win.winfo_height()
    x = (win.winfo_screenwidth() // 2) - (width // 2)
    y = (win.winfo_screenheight() // 2) - (height*2)
    win.geometry('{}x{}+{}+{}'.format(width, height, x, y))
def shift(seq):
    l = seq[1:]
    l.append(seq[0])
    return l
def pg():
    row_frame = Frame(fr)
    row_frame.pack(side="top")
    playag = bt(row_frame, text="Play Again")
    playag.config(command=playag.play)
    playag.pack()
def playagain():
    global fr
    global won
    global n
    global w
    global btn
    global num
    num = 1
    window.geometry("200x200")
    fr.destroy()
    won = False
    n = "x"
    fr = Frame(window)
    fr.pack()
    BoardValue = ["-","-","-","-","-","-","-","-","-"]
    w = Label(fr, text="Noughts And Crosses")
    w.pack()
    btn = []
    for i, b in enumerate(BoardValue):
        if i%3 == 0:
            row_frame = Frame(fr)
            row_frame.pack(side="top")
        btn.append(bt(row_frame, text=b, relief=GROOVE, width=4,height=2))
        btn[i].config(command=btn[i].x)
        btn[i].nam(str(i//3)+str(i%3))
        btn[i].pack(side="left")

#AI CLASS
class AI:
    def checkNear(self,let):
        global btn
        global wins
        for j in wins:
            n1 = (int(j[0])*3) + int(j[1])
            n2 = (int(j[2])*3) + int(j[3])
            n3 = (int(j[4])*3) + int(j[5])
            btns = [btn[n1],btn[n2],btn[n3]]
            for i in range(3):
                if (btns[0].gt() == btns[1].gt()) and (btns[2].gt() == "-") and (btns[1].gt() == let):
                    return btns[2].nams
                    break
                btns = shift(btns)

    def firstmove(self):
        global btn
        if btn[4].cget('state') == 'normal':
            return "11"
        else:
            return "00"

    def order(self):
        global btn
        for x in [1,3,5,7,0,2,6,8,4]:
                if btn[x].cget('state') == 'normal':
                    return str(x//3)+str(x%3)

    def diag(self):
        global btn
        ps = {
            '00': [[1,3],[2,3],[1,6]],
            '02': [[1,5],[2,5],[2,8]],
            '20': [[3,7],[3,8],[0,7]],
            '22': [[5,7],[2,7],[5,6]]
            }
        for a in ps:
            for b in ps[a]:
                if btn[b[0]].gt() == btn[b[1]].gt() and btn[b[0]].gt() == "X":
                    if btn[(int(a[0])*3) + int(a[1])].cget('state') == 'normal':
                        return a

    def miderr(self):
        global btn
        if btn[4].gt() == btn[8].gt() and btn[4].gt() == "X":
            return '02'

    def move(self,n):
        global btn
        if n == 1:
            return self.firstmove()
        else:
            ms = []
            co = self.checkNear("O")
            ms.append(co)
            cx = self.checkNear("X")
            ms.append(cx)
            if n == 2:
                m = self.miderr()
                ms.append(m)
            if n <= 3:
                d = self.diag()
                ms.append(d)
            o = self.order()
            ms.append(o)
            return next(x for x in ms if x != None)
#END AI CLASS

ai = AI() #defining game AI for 1-player game
#switch functions
def switch1():
    global ai
    global n
    global num
    global btn
    if n == "x":
        for i in btn:
            i.config(command=i.o)
        n = "o"
        try:
            mov = ai.move(num)
            num += 1
            but = (int(mov[0])*3) + int(mov[1])
            btn[but].invoke()
        except:
            None
    else:
        for i in btn:
            i.config(command=i.x)
        n = "x"
def switch2():
    global n
    if n == "x":
        for i in btn:
            i.config(command=i.o)
        n = "o"
    else:
        for i in btn:
            i.config(command=i.x)
        n = "x"
switch = switch2
#more functions
def checkEqual(iterator):
    try:
        return all('disabled' == rest.cget('state') for rest in iterator)
    except StopIteration:
        return True
def chw():
    global btn
    global won
    global w
    for j in wins:
        n1 = (int(j[0])*3) + int(j[1])
        n2 = (int(j[2])*3) + int(j[3])
        n3 = (int(j[4])*3) + int(j[5])
        btns = [btn[n1],btn[n2],btn[n3]]
        if (btns[0].gt() == btns[1].gt()) and (btns[2].gt() == btns[1].gt()) and (btns[0].gt() != "-") and won == False:
            for a in btns:
                a.config(bg="red", disabledforeground="white")
            win = "Noughts and Crosses - %s wins!" %(btns[0].gt())
            won = True
            w.config(text=win)
            for x in btn:
                x.config(state="disabled")
            pg()
    if checkEqual(btn) and not(won):
        w.config(text="Noughts and Crosses - It's a draw!")
        pg()
#modified button class
class bt(Button):
    def p1(self):
        global switch
        switch = switch1
        playagain()
    def p2(self):
        global switch
        switch = switch2
        playagain()
    def play(self):
        playagain()
    def gt(self):
        return self.cget('text')
    def nam(self,nam):
        self.nams = nam
    def x(self):
        self.config(text="X",state="disabled")
        chw()
        switch()
    def o(self):
        self.config(text="O",state="disabled")
        switch()
        chw()
#setup window
BoardValue = ["-","-","-","-","-","-","-","-","-"]
window.title("Noughts And Crosses")
window.geometry("200x100")
window.wm_iconbitmap("tic-tac-toe.ico")
centre(window)
w.pack()
#setup gamemode choice
play1 = bt(fr,text="1-player")
play2 = bt(fr,text="2-player")
play1.config(command=play1.p1)
play2.config(command=play1.p2)
play1.pack(fill=X)
play2.pack(fill=X)
window.mainloop()
<|endoftext|>


<commit_before># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright 2010 OpenStack LLC
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import commands
import errno
import glob
import os
import select

from eventlet import greenpool
from eventlet import greenthread
import lockfile

from nova import exception
from nova import test
from nova import utils

class ExceptionTestCase(test.TestCase):
    @staticmethod
    def _raise_exc(exc):
        raise exc()

    def test_exceptions_raise(self):
        for name in dir(exception):
            exc = getattr(exception, name)
            if isinstance(exc, type):
                self.assertRaises(exc, self._raise_exc, exc)

class ProjectTestCase(test.TestCase):

    def test_all_migrations_have_downgrade(self):
        topdir = os.path.normpath(os.path.dirname(__file__) + '/../../')
        py_glob = os.path.join(topdir, "nova", "db", "sqlalchemy",
                               "migrate_repo", "versions", "*.py")
        missing_downgrade = []
        for path in glob.iglob(py_glob):
            has_upgrade = False
            has_downgrade = False
            with open(path, "r") as f:
                for line in f:
                    if 'def upgrade(' in line:
                        has_upgrade = True
                    if 'def downgrade(' in line:
                        has_downgrade = True

                if has_upgrade and not has_downgrade:
                    fname = os.path.basename(path)
                    missing_downgrade.append(fname)

        helpful_msg = (_("The following migrations are missing a downgrade:"
                         "\n\t%s") % '\n\t'.join(sorted(missing_downgrade)))
        self.assert_(not missing_downgrade, helpful_msg)

class LockTestCase(test.TestCase):
    def test_synchronized_wrapped_function_metadata(self):
        @utils.synchronized('whatever')
        def foo():
            """Bar"""
            pass
        self.assertEquals(foo.__doc__, 'Bar', "Wrapped function's docstring "
                                              "got lost")
        self.assertEquals(foo.__name__, 'foo', "Wrapped function's name "
                                               "got mangled")

    def test_synchronized_internally(self):
        """We can lock across multiple green threads"""
        saved_sem_num = len(utils._semaphores)
        seen_threads = list()

        @utils.synchronized('testlock2', external=False)
        def f(id):
            for x in range(10):
                seen_threads.append(id)
                greenthread.sleep(0)

        threads = []
        pool = greenpool.GreenPool(10)
        for i in range(10):
            threads.append(pool.spawn(f, i))

        for thread in threads:
            thread.wait()

        self.assertEquals(len(seen_threads), 100)
        # Looking at the seen threads, split it into chunks of 10, and verify
        # that the last 9 match the first in each chunk.
        for i in range(10):
            for j in range(9):
                self.assertEquals(seen_threads[i * 10],
                                  seen_threads[i * 10 + 1 + j])

        self.assertEqual(saved_sem_num, len(utils._semaphores),
                         "Semaphore leak detected")

    def test_nested_external_fails(self):
        """We can not nest external syncs"""

        @utils.synchronized('testlock1', external=True)
        def outer_lock():

            @utils.synchronized('testlock2', external=True)
            def inner_lock():
                pass
            inner_lock()
        try:
            self.assertRaises(lockfile.NotMyLock, outer_lock)
        finally:
            utils.cleanup_file_locks()

    def test_synchronized_externally(self):
        """We can lock across multiple processes"""
        rpipe1, wpipe1 = os.pipe()
        rpipe2, wpipe2 = os.pipe()

        @utils.synchronized('testlock1', external=True)
        def f(rpipe, wpipe):
            try:
                os.write(wpipe, "foo")
            except OSError, e:
                self.assertEquals(e.errno, errno.EPIPE)
                return

            rfds, _wfds, _efds = select.select([rpipe], [], [], 1)
            self.assertEquals(len(rfds), 0, "The other process, which was"
                                            " supposed to be locked, "
                                            "wrote on its end of the "
                                            "pipe")
            os.close(rpipe)

        pid = os.fork()
        if pid > 0:
            os.close(wpipe1)
            os.close(rpipe2)

            f(rpipe1, wpipe2)
        else:
            os.close(rpipe1)
            os.close(wpipe2)

            f(rpipe2, wpipe1)
            os._exit(0)
<commit_msg>Remove unused imports<commit_after># vim: tabstop=4 shiftwidth=4 softtabstop=4

#    Copyright 2010 OpenStack LLC
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import errno
import glob
import os
import select

from eventlet import greenpool
from eventlet import greenthread
import lockfile

from nova import exception
from nova import test
from nova import utils

class ExceptionTestCase(test.TestCase):
    @staticmethod
    def _raise_exc(exc):
        raise exc()

    def test_exceptions_raise(self):
        for name in dir(exception):
            exc = getattr(exception, name)
            if isinstance(exc, type):
                self.assertRaises(exc, self._raise_exc, exc)

class ProjectTestCase(test.TestCase):

    def test_all_migrations_have_downgrade(self):
        topdir = os.path.normpath(os.path.dirname(__file__) + '/../../')
        py_glob = os.path.join(topdir, "nova", "db", "sqlalchemy",
                               "migrate_repo", "versions", "*.py")
        missing_downgrade = []
        for path in glob.iglob(py_glob):
            has_upgrade = False
            has_downgrade = False
            with open(path, "r") as f:
                for line in f:
                    if 'def upgrade(' in line:
                        has_upgrade = True
                    if 'def downgrade(' in line:
                        has_downgrade = True

                if has_upgrade and not has_downgrade:
                    fname = os.path.basename(path)
                    missing_downgrade.append(fname)

        helpful_msg = (_("The following migrations are missing a downgrade:"
                         "\n\t%s") % '\n\t'.join(sorted(missing_downgrade)))
        self.assert_(not missing_downgrade, helpful_msg)

class LockTestCase(test.TestCase):
    def test_synchronized_wrapped_function_metadata(self):
        @utils.synchronized('whatever')
        def foo():
            """Bar"""
            pass
        self.assertEquals(foo.__doc__, 'Bar', "Wrapped function's docstring "
                                              "got lost")
        self.assertEquals(foo.__name__, 'foo', "Wrapped function's name "
                                               "got mangled")

    def test_synchronized_internally(self):
        """We can lock across multiple green threads"""
        saved_sem_num = len(utils._semaphores)
        seen_threads = list()

        @utils.synchronized('testlock2', external=False)
        def f(id):
            for x in range(10):
                seen_threads.append(id)
                greenthread.sleep(0)

        threads = []
        pool = greenpool.GreenPool(10)
        for i in range(10):
            threads.append(pool.spawn(f, i))

        for thread in threads:
            thread.wait()

        self.assertEquals(len(seen_threads), 100)
        # Looking at the seen threads, split it into chunks of 10, and verify
        # that the last 9 match the first in each chunk.
        for i in range(10):
            for j in range(9):
                self.assertEquals(seen_threads[i * 10],
                                  seen_threads[i * 10 + 1 + j])

        self.assertEqual(saved_sem_num, len(utils._semaphores),
                         "Semaphore leak detected")

    def test_nested_external_fails(self):
        """We can not nest external syncs"""

        @utils.synchronized('testlock1', external=True)
        def outer_lock():

            @utils.synchronized('testlock2', external=True)
            def inner_lock():
                pass
            inner_lock()
        try:
            self.assertRaises(lockfile.NotMyLock, outer_lock)
        finally:
            utils.cleanup_file_locks()

    def test_synchronized_externally(self):
        """We can lock across multiple processes"""
        rpipe1, wpipe1 = os.pipe()
        rpipe2, wpipe2 = os.pipe()

        @utils.synchronized('testlock1', external=True)
        def f(rpipe, wpipe):
            try:
                os.write(wpipe, "foo")
            except OSError, e:
                self.assertEquals(e.errno, errno.EPIPE)
                return

            rfds, _wfds, _efds = select.select([rpipe], [], [], 1)
            self.assertEquals(len(rfds), 0, "The other process, which was"
                                            " supposed to be locked, "
                                            "wrote on its end of the "
                                            "pipe")
            os.close(rpipe)

        pid = os.fork()
        if pid > 0:
            os.close(wpipe1)
            os.close(rpipe2)

            f(rpipe1, wpipe2)
        else:
            os.close(rpipe1)
            os.close(wpipe2)

            f(rpipe2, wpipe1)
            os._exit(0)
<|endoftext|>