<commit_before>#!/usr/bin/python
"""
Cuteness Delivery System.

This program requests the top links from various subreddits of cute animals
and email them to participants.
"""
from __future__ import unicode_literals, absolute_import
# try:
#     from html import escape
#     print('Python 3: using html.escape()')
# except ImportError:
#     from cgi import escape
#     print('Python 2: using cgi.escape()')

import re
import sys
import praw
import django
import smtplib
import calendar
import requests
from pytz import timezone
from itertools import chain
from bs4 import BeautifulSoup
from operator import attrgetter
from datetime import date, datetime
# from email.mime.text import MIMEText
# from email.mime.multipart import MIMEMultipart
from constants import LIMIT, EMAIL, APP_PASSWORD
django.setup()
from django.conf import settings
from subscribers.models import Subscriber
from django.core.mail import send_mail
from django.template.loader import render_to_string

USER_AGENT = 'python:deliver_cute:v1.0 (by /u/____OOOO____)'

EMAIL_SUBJECT_TEMPLATE = '{debug}Cute Pics for {date}'
FROM_NAME = 'Deliver Cute'
PIC_WIDTH = '400'

YT_PAT = re.compile(r'.*(youtu\.be|youtube\.com).*')
SRC_PAT = re.compile(r'http(s)?://i\.(imgur|reddituploads|redd).*\.[a-z]{3,4}')

def main(debug):
    """Gather then email top cute links."""
    subscribers = subscribers_for_now(debug)
    if not subscribers:
        now = datetime.now(tz=timezone('US/Pacific'))
        print('No subscribers want cute delivered at {}'.format(now.hour))
        return 0

    subreddit_names = chain(*(s.subreddit_names() for s in subscribers))
    post_map = create_post_map(subreddit_names, LIMIT)

    # server = setup_email_server(EMAIL, APP_PASSWORD)
    subject = get_email_subject(debug)

    sent_count = 0
    for subscriber in subscribers:
        posts = get_relevant_posts(post_map, subscriber)
        posts = fix_image_links(posts)
        posts = dedupe_posts(posts)
        posts = sort_posts(posts)
        posts = htmlize_posts(posts)
        body = get_email_body(subscriber, posts)
        sent_count += send_mail(
            subject, 'DEBUG', EMAIL, [subscriber.email],
            html_message=body,
            fail_silently=False,
            # server, EMAIL, FROM_NAME, subscriber.email, subject, body
        )

    # server.quit()
    return sent_count

def subscribers_for_now(debug):
    """Collect subscribers with send_hour set to current time."""
    if debug:
        return Subscriber.objects.filter(email=EMAIL)
    now = datetime.now(tz=timezone('US/Pacific'))
    return Subscriber.objects.filter(send_hour=now.hour)

def create_post_map(subreddit_names, limit):
    """Return dictionary with keys of subreddit names at given post limit."""
    reddit = praw.Reddit(user_agent=USER_AGENT)
    post_map = dict.fromkeys(subreddit_names)
    for name in post_map:
        subreddit = reddit.get_subreddit(name)
        new_posts = subreddit.get_top_from_day(limit=limit)
        post_map[name] = list(new_posts)
    return post_map

def get_relevant_posts(post_map, subscriber):
    """Filter only those posts selected by the current subscriber."""
    for subreddit_name in subscriber.subreddit_names():
        for post in post_map[subreddit_name]:
            yield post

def dedupe_posts(posts):
    """Generate posts where duplicates have been removed by comparing url."""
    found_already = set()
    for post in posts:
        if post.url not in found_already:
            yield post
            found_already.add(post.url)
        else:
            print('Omitting duplicate {}'.format(post.url))

def sort_posts(posts):
    """Generate posts sorted by their upvote count."""
    for post in sorted(posts, key=attrgetter('score'), reverse=True):
        yield post

def fix_image_links(posts):
    """Make sure that each imgur link is directly to the content."""
    for post in posts:
        link = post.url
        if YT_PAT.match(link):
            continue

        # Temporary measure until able to display gifv and gyfcat properly
        if link.endswith('gifv') or link.endswith('mp4') or 'gfycat' in link:
            continue

        if not SRC_PAT.match(link):
            try:
                link = find_source_link(link)
            except AttributeError as e:
                print('Error trying to get img src at {}: {}'.format(link, e))
                continue
        link = re.sub(r'^//', 'http://', link)
        post.url = link
        yield post

def find_source_link(link):
    """Scrape the direct source link from imgur or other website."""
    # Currently only works for imgur
    response = requests.get(link)
    html = BeautifulSoup(response.text, 'html.parser')
    div = html.find('div', class_='post-image')
    img = div.find('img')
    return img.attrs['src']

def htmlize_posts(posts):
    """Generate each link as an html-ized image element."""
    for post in posts:
        subreddit = post.subreddit.display_name
        context = {
            'subreddit': subreddit,
            'subreddit_name': '/r/' + subreddit,
            'subreddit_url': 'https://www.reddit.com/r/' + subreddit,
            'title': post.title,
            'url': post.url,
            'permalink': post.permalink,
            'width': PIC_WIDTH,
        }
        yield render_to_string('image.html', context=context)

def get_email_body(subscriber, posts):
    """Format posts into HTML."""
    context = {
        'posts': posts,
        'subscriber': subscriber,
        'site_url': settings.SITE_URL,
    }
    return render_to_string('daily_email.html', context=context)

def get_email_subject(debug):
    """Format today's date into the email subject."""
    today = date.today()
    day_name = calendar.day_name[today.weekday()]
    month_name = calendar.month_name[today.month]
    today_date_str = '{d}, {m} {i} {y}'.format(
        d=day_name,
        m=month_name,
        i=today.day,
        y=today.year,
    )
    return EMAIL_SUBJECT_TEMPLATE.format(
        debug='DEBUG ' * debug,
        date=today_date_str)

def setup_email_server(email, password):
    """Send an email using gmail's smtp server."""
    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.ehlo()
    server.starttls()
    server.login(email, password)
    return server

# def send_email(server, from_addr, from_name, to_addr, subject, body):
#     """Send an email with given server and message info."""
#     html = MIMEText(body, 'html')
#     msg = MIMEMultipart('alternative')
#     msg.attach(html)
#     msg['Subject'] = subject
#     msg['From'] = from_name
#     msg['To'] = to_addr
#     server.sendmail(from_addr, to_addr, msg.as_string())
#     print('Email send to {}'.format(to_addr))

if __name__ == '__main__':
    try:
        debug = bool(sys.argv[1])
    except IndexError:
        debug = False
    print('Debug is {}'.format(debug))
    print('{} email sent.'.format(main(debug)))
<commit_msg>First steps on on_schedule refactor.<commit_after>#!/usr/bin/python
"""
Cuteness Delivery System.

This program requests the top links from various subreddits of cute animals
and email them to participants.
"""
from __future__ import unicode_literals, absolute_import
# try:
#     from html import escape
#     print('Python 3: using html.escape()')
# except ImportError:
#     from cgi import escape
#     print('Python 2: using cgi.escape()')

import re
import sys
import praw
import django
import smtplib
import calendar
import requests
from pytz import timezone
from itertools import chain
from bs4 import BeautifulSoup
from operator import attrgetter
from datetime import date, datetime
# from email.mime.text import MIMEText
# from email.mime.multipart import MIMEMultipart
from constants import LIMIT, EMAIL, APP_PASSWORD
django.setup()
from django.conf import settings
from subscribers.models import Subscriber
from django.core.mail import send_mail
from django.template.loader import render_to_string

USER_AGENT = 'python:deliver_cute:v1.0 (by /u/____OOOO____)'

EMAIL_SUBJECT_TEMPLATE = '{debug}Cute Pics for {date}'
FROM_NAME = 'Deliver Cute'
PIC_WIDTH = '400'

YT_PAT = re.compile(r'.*(youtu\.be|youtube\.com).*')
SRC_PAT = re.compile(r'http(s)?://i\.(imgur|reddituploads|redd).*\.[a-z]{3,4}')

def main(debug):
    """Gather then email top cute links."""
    subscribers = subscribers_for_now(debug)
    if not subscribers:
        now = datetime.now(tz=timezone('US/Pacific'))
        print('No subscribers want cute delivered at {}'.format(now.hour))
        return 0

    # subreddit_names = chain(*(s.subreddit_names() for s in subscribers))
    # post_map = create_post_map(subreddit_names, LIMIT)

    # server = setup_email_server(EMAIL, APP_PASSWORD)
    subject = get_email_subject(debug)

    reddit = praw.Reddit(user_agent=USER_AGENT)

    sent_count = 0
    post_map = {}
    found_posts = set()
    for subscriber in subscribers:
        subreddit_names = set(subscriber.subreddit_names()) - post_map.keys()
        # posts = get_relevant_posts(post_map, subscriber)
        posts = get_posts_from_reddit(reddit, subreddit_names, LIMIT)
        posts = fix_image_links(posts)
        posts = dedupe_posts(posts)
        posts = sort_posts(posts)
        posts = htmlize_posts(posts)
        body = get_email_body(subscriber, posts)
        sent_count += send_mail(
            subject, 'DEBUG', EMAIL, [subscriber.email],
            html_message=body,
            fail_silently=False,
            # server, EMAIL, FROM_NAME, subscriber.email, subject, body
        )

    # server.quit()
    return sent_count

def subscribers_for_now(debug):
    """Collect subscribers with send_hour set to current time."""
    if debug:
        return Subscriber.objects.filter(email=EMAIL)
    now = datetime.now(tz=timezone('US/Pacific'))
    return Subscriber.objects.filter(send_hour=now.hour)

def get_posts_from_reddit(reddit, subreddit_names, limit):
    """Get subreddit names from given subreddit name."""
    subreddit = reddit.get_subreddit(name)
    return subreddit.get_top_from_day(limit=limit)

def create_post_map(subreddit_names, limit):
    """Return dictionary with keys of subreddit names at given post limit."""
    reddit = praw.Reddit(user_agent=USER_AGENT)
    post_map = dict.fromkeys(subreddit_names)
    for name in post_map:
        subreddit = reddit.get_subreddit(name)
        new_posts = subreddit.get_top_from_day(limit=limit)
        post_map[name] = list(new_posts)
    return post_map

def get_relevant_posts(post_map, subscriber):
    """Filter only those posts selected by the current subscriber."""
    for subreddit_name in subscriber.subreddit_names():
        for post in post_map[subreddit_name]:
            yield post

def dedupe_posts(posts):
    """Generate posts where duplicates have been removed by comparing url."""
    found_already = set()
    for post in posts:
        if post.url not in found_already:
            yield post
            found_already.add(post.url)
        else:
            print('Omitting duplicate {}'.format(post.url))

def sort_posts(posts):
    """Generate posts sorted by their upvote count."""
    for post in sorted(posts, key=attrgetter('score'), reverse=True):
        yield post

def fix_image_links(posts):
    """Make sure that each imgur link is directly to the content."""
    for post in posts:
        link = post.url
        if YT_PAT.match(link):
            continue

        # Temporary measure until able to display gifv and gyfcat properly
        if link.endswith('gifv') or link.endswith('mp4') or 'gfycat' in link:
            continue

        if not SRC_PAT.match(link):
            try:
                link = find_source_link(link)
            except AttributeError as e:
                print('Error trying to get img src at {}: {}'.format(link, e))
                continue
        link = re.sub(r'^//', 'http://', link)
        post.url = link
        yield post

def find_source_link(link):
    """Scrape the direct source link from imgur or other website."""
    # Currently only works for imgur
    response = requests.get(link)
    html = BeautifulSoup(response.text, 'html.parser')
    div = html.find('div', class_='post-image')
    img = div.find('img')
    return img.attrs['src']

def htmlize_posts(posts):
    """Generate each link as an html-ized image element."""
    for post in posts:
        subreddit = post.subreddit.display_name
        context = {
            'subreddit': subreddit,
            'subreddit_name': '/r/' + subreddit,
            'subreddit_url': 'https://www.reddit.com/r/' + subreddit,
            'title': post.title,
            'url': post.url,
            'permalink': post.permalink,
            'width': PIC_WIDTH,
        }
        yield render_to_string('image.html', context=context)

def get_email_body(subscriber, posts):
    """Format posts into HTML."""
    context = {
        'posts': posts,
        'subscriber': subscriber,
        'site_url': settings.SITE_URL,
    }
    return render_to_string('daily_email.html', context=context)

def get_email_subject(debug):
    """Format today's date into the email subject."""
    today = date.today()
    day_name = calendar.day_name[today.weekday()]
    month_name = calendar.month_name[today.month]
    today_date_str = '{d}, {m} {i} {y}'.format(
        d=day_name,
        m=month_name,
        i=today.day,
        y=today.year,
    )
    return EMAIL_SUBJECT_TEMPLATE.format(
        debug='DEBUG ' * debug,
        date=today_date_str)

def setup_email_server(email, password):
    """Send an email using gmail's smtp server."""
    server = smtplib.SMTP('smtp.gmail.com', 587)
    server.ehlo()
    server.starttls()
    server.login(email, password)
    return server

# def send_email(server, from_addr, from_name, to_addr, subject, body):
#     """Send an email with given server and message info."""
#     html = MIMEText(body, 'html')
#     msg = MIMEMultipart('alternative')
#     msg.attach(html)
#     msg['Subject'] = subject
#     msg['From'] = from_name
#     msg['To'] = to_addr
#     server.sendmail(from_addr, to_addr, msg.as_string())
#     print('Email send to {}'.format(to_addr))

if __name__ == '__main__':
    try:
        debug = bool(sys.argv[1])
    except IndexError:
        debug = False
    print('Debug is {}'.format(debug))
    print('{} email sent.'.format(main(debug)))
<|endoftext|>


<commit_before># Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import time
import socket
import os
import six
import logging
import signal
import random

logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper())
logger = logging.getLogger("ELASTIC")

ELASTIC_EXIT_CODE = 101

class ElasticStatus:
    COMPLETED = "completed"
    ERROR = "error"
    HOLD = "hold"
    RESTART = "restart"
    EXIT = "exit"

class LauncherInterface(object):
    def __init__(self, args):
        self.args = args
        self.procs = []

    def _terminate_procs(self):
        # try to terminate process by group, this happend in multiprocess senario in user process
        if os.name != 'nt':
            for p in self.procs:
                if p.proc.poll() is None:
                    os.killpg(os.getpgid(p.proc.pid), signal.SIGTERM)
                    if p.log_fn:
                        p.log_fn.close()
                    logger.info("terminate process group gid:{}".format(
                        p.proc.pid))

            time.sleep(1)
        for p in self.procs:
            if p.proc.poll() is None:
                p.proc.terminate()
                if p.log_fn:
                    p.log_fn.close()
                logger.info("terminate process id:{}".format(p.proc.pid))

        for step in range(0, 50):
            alive = False
            for p in self.procs:
                if p.proc.poll() is None:  # not termniate
                    os.kill(p.proc.pid, signal.SIGKILL)
                    alive = True

            if not alive:
                logger.info("terminated all the procs")
                return True

            time.sleep(1)
        return False

    def _check_procs(self):
        alive = False
        result = None
        for p in self.procs:
            ret = p.proc.poll()
            if ret is None:
                alive = True
            elif ret != 0:
                logger.error("ABORT!!! ABORT!!! ABORT!!!")
                logger.error(
                    "ERROR rank {} error with exit code {}, check log for detail.".
                    format(p.rank, ret))
                result = ret
        if not alive and result is None:
            return 0
        else:
            return result

    def launch(self):
        raise NotImplementedError

    def stop(self):
        raise NotImplementedError

    def watch(self):
        raise NotImplementedError

class ElasticManager(object):
    def __init__(self, args):

        self.args = args
        server = args.elastic_server or os.getenv('PADDLE_ELASTIC_SERVER')
        name = args.job_id or os.getenv('PADDLE_ELASTIC_JOB_ID')
        np = args.np or int(os.getenv('PADDLE_ELASTIC_NP', 0))
        host = args.host or os.getenv('POD_IP')
        scale = args.scale or int(os.getenv('PADDLE_ELASTIC_SCALE', 0))
        force = args.force or os.getenv('PADDLE_ELASTIC_FORCE')

        self.endpoints = os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS', '')
        self.trainers = os.getenv('PADDLE_TRAINERS', '')

        self.elastic_level = int(
            os.getenv('PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL', 1))

        # compatible with kuberntes service discovery
        if not server and os.getenv(
                'PADDLE_ELASTIC_ETCD_SERVICE_HOST') and os.getenv(
                    'PADDLE_ELASTIC_ETCD_SERVICE_PORT'):
            server = '{}:{}'.format(
                os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_HOST'),
                os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_PORT'))

        #elastic_timeout = os.getenv('PADDLE_ELASTIC_TIMEOUT',1)

        logger.debug('init with server {} host {}'.format(server, host))

        self.hosts = []
        self.stopped = False

        self.sigint = 0

        if not server or ':' not in server or not name or not np:
            logger.info(
                'Elastic is not enabled with server {} name {} and np {}'.
                format(server, name, np))
            self.enable = False
            return
        else:
            self.enable = True

        import etcd3

        srv, port = server.split(':')
        self.etcd = etcd3.client(host=srv, port=port)
        self.host = host if host else self._get_host()

        # etcd data
        self.prefix = "/paddle/" + name
        self.node_prefix = self.prefix + '/nodes'
        self.np_path = self.prefix + '/np'
        self.endpoints_path = self.prefix + '/endpoints'

        node_tag = ''.join(
            random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6))
        self.host_path = '{}/{}{}'.format(self.node_prefix, node_tag,
                                          time.time())

        self.np = np + scale
        '''
        0 group mode, be aware of healthy status of other workers
        1 decouple mode, check own status only
        '''
        self.etcd.put(self.prefix, b'0')

        # host
        # register self host to etcd
        # register watch to reset host after host been deleted
        self.etcd.delete_prefix(self.node_prefix)

        def host_call_back(event):
            if self.etcd.get(self.host_path)[0] == None:
                logger.info('register host again {}'.format(self.host))

                self.etcd.put(self.host_path, six.b(self.host))

        host_watch = self.etcd.add_watch_callback(self.host_path,
                                                  host_call_back)
        self.etcd.put(self.host_path, six.b(self.host))

        # np describes the exact number of nodes to run the job
        inp = int(self.etcd.get(self.np_path)[0] or 0)
        if scale == 0 and not force:
            assert inp == np or inp == 0, "np {} is not consistent with np in etcd {}".format(
                np, inp)
        else:
            assert inp == np or inp == self.np, "np {} scale to {} by {} is not allowed".format(
                inp, self.np, scale)

        self.etcd.put(self.np_path, six.b("%d" % (self.np)))

        def np_call_back(event):
            gnp = int(self.etcd.get(self.np_path)[0])
            if gnp != self.np:
                logger.info("scale np {} to {} ".format(self.np, gnp))
                self.np = gnp

        np_watch = self.etcd.add_watch_callback(self.np_path, np_call_back)

        # endpoints handle DISTRIBUTED_TRAINER_ENDPOINTS and PADDLE_TRAINERS
        self.etcd.put(self.endpoints_path,
                      six.b('{}|{}'.format(self.endpoints, self.trainers)))

        def endpoints_call_back(event):
            if not self.endpoints:
                return
            edps = six.ensure_str(self.etcd.get(self.endpoints_path)[0] or '')
            self.endpoints, self.trainers = edps.split('|')
            logger.info("set DISTRIBUTED_TRAINER_ENDPOINTS {} ".format(
                self.endpoints))
            logger.info("set PADDLE_TRAINERS {} ".format(self.trainers))

        endpoints_watch = self.etcd.add_watch_callback(self.endpoints_path,
                                                       endpoints_call_back)

        self.watches = [host_watch, np_watch, endpoints_watch]

        self.launcher = None

    def exit(self, completed=False):
        logger.info('manager exist completed {}'.format(completed))

        if self.launcher:
            self.launcher.stop()

        if not self.enable:
            return

        if completed:
            self.etcd.put(self.prefix, b'1')

        for watch in self.watches:
            self.etcd.cancel_watch(watch)
        self.etcd.delete(self.host_path)

        hosts = [i for i in self.etcd.get_prefix(self.node_prefix)]
        if len(hosts) == 0:
            self.etcd.delete_prefix(self.prefix)

    def _get_host(self):
        try:
            return socket.gethostbyname(socket.getfqdn(socket.gethostname()))
        except:
            return '127.0.0.1'

    def _completed(self):
        if not self.enable:
            return True

        return int(self.etcd.get(self.prefix)[0]) == 1

    def _match(self):
        self.hosts = [
            six.ensure_str(i[0]) for i in self.etcd.get_prefix(self.node_prefix)
        ]
        if len(self.hosts) == self.np:
            return True
        else:
            return False

    def _update_hosts(self):
        assert len(self.hosts) != 0, 'hosts empty'

        if self.host in self.endpoints:
            os.environ['DISTRIBUTED_TRAINER_ENDPOINTS'] = self.endpoints
            os.environ['PADDLE_TRAINERS'] = self.trainers
            logger.info("update env DISTRIBUTED_TRAINER_ENDPOINTS {} ".format(
                self.endpoints))
            logger.info("update env PADDLE_TRAINERS {} ".format(self.trainers))
            return

        rank = int(os.getenv('PADDLE_TRAINER_ID', -1))
        idx = self.hosts.index(self.host)

        # swap if self.host not in the right position
        if rank >= 0:
            self.hosts[idx] = self.hosts[rank]
            self.hosts[rank] = self.host
        else:
            os.environ['PADDLE_TRAINER_ID'] = '{}'.format(idx)

        hosts = ','.join(self.hosts)
        self.args.ips = hosts
        os.environ['PADDLE_TRAINERS'] = hosts

    def wait(self):
        if not self.enable:
            return

        idx = 1
        while not self.stopped:
            if self._match():
                logger.info('ready with hosts {}'.format(self.hosts))
                self._update_hosts()
                return
            logger.info('not ready for np {} with hosts {}'.format(self.np,
                                                                   self.hosts))

            # reset hosts every 30s to prevent fake deadlock
            if idx % 10 == 0:
                self.etcd.delete_prefix(self.node_prefix)
                logger.info('reset np {} with hosts {}'.format(self.np,
                                                               self.hosts))

            idx += 1
            time.sleep(3)
        return

    def run(self, launcher):
        if self.stopped:
            return

        self.launcher = launcher(self.args)
        self.launcher.launch()

    def watch(self):

        while not self.stopped:
            ret = self.launcher.watch()

            if ret is not None:  # self terminated
                logger.info('job exit with code {}'.format(ret))
                # process is completed if ret >= 0 or error else
                completed = True if ret == 0 else False
                self.exit(completed=completed)
                if completed:
                    return ElasticStatus.COMPLETED
                if self.elastic_level == 1:
                    return ElasticStatus.RESTART
                else:
                    return ElasticStatus.ERROR

            if not self._completed() and not self._match():
                self.launcher.stop()
                return ElasticStatus.HOLD

            time.sleep(2)

        if self.launcher:
            self.launcher.stop()
        return ElasticStatus.EXIT

    def signal_handler(self, sigint, frame):
        if self.enable:
            self.exit()
        self.sigint = sigint
        self.stopped = True
<commit_msg>fix force kill for elastic (#34488)<commit_after># Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import time
import socket
import os
import six
import logging
import signal
import random

logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper())
logger = logging.getLogger("ELASTIC")

ELASTIC_EXIT_CODE = 101

class ElasticStatus:
    COMPLETED = "completed"
    ERROR = "error"
    HOLD = "hold"
    RESTART = "restart"
    EXIT = "exit"

class LauncherInterface(object):
    def __init__(self, args):
        self.args = args
        self.procs = []

    def _terminate_procs(self):
        # try to terminate process by group, this happend in multiprocess senario in user process
        if os.name != 'nt':
            for p in self.procs:
                if p.proc.poll() is None:
                    os.killpg(os.getpgid(p.proc.pid), signal.SIGTERM)
                    if p.log_fn:
                        p.log_fn.close()
                    logger.info("terminate process group gid:{}".format(
                        p.proc.pid))

            time.sleep(1)
        for p in self.procs:
            if p.proc.poll() is None:
                p.proc.terminate()
                if p.log_fn:
                    p.log_fn.close()
                logger.info("terminate process id:{}".format(p.proc.pid))

        for step in range(0, 50):
            alive = False
            for p in self.procs:
                if p.proc.poll() is None:  # not termniate
                    os.kill(p.proc.pid, signal.SIGKILL)
                    alive = True

            if not alive:
                logger.info("terminated all the procs")
                return True

            time.sleep(1)
        return False

    def _check_procs(self):
        alive = False
        result = None
        for p in self.procs:
            ret = p.proc.poll()
            if ret is None:
                alive = True
            elif ret != 0:
                logger.error("ABORT!!! ABORT!!! ABORT!!!")
                logger.error(
                    "ERROR rank {} error with exit code {}, check log for detail.".
                    format(p.rank, ret))
                result = ret
        if not alive and result is None:
            return 0
        else:
            return result

    def launch(self):
        raise NotImplementedError

    def stop(self):
        raise NotImplementedError

    def watch(self):
        raise NotImplementedError

class ElasticManager(object):
    def __init__(self, args):

        self.args = args
        server = args.elastic_server or os.getenv('PADDLE_ELASTIC_SERVER')
        name = args.job_id or os.getenv('PADDLE_ELASTIC_JOB_ID')
        np = args.np or int(os.getenv('PADDLE_ELASTIC_NP', 0))
        host = args.host or os.getenv('POD_IP')
        scale = args.scale or int(os.getenv('PADDLE_ELASTIC_SCALE', 0))
        force = args.force or os.getenv('PADDLE_ELASTIC_FORCE')

        self.endpoints = os.getenv('DISTRIBUTED_TRAINER_ENDPOINTS', '')
        self.trainers = os.getenv('PADDLE_TRAINERS', '')

        self.elastic_level = int(
            os.getenv('PADDLE_ELASTIC_FAULT_TOLERANC_LEVEL', 1))

        # compatible with kuberntes service discovery
        if not server and os.getenv(
                'PADDLE_ELASTIC_ETCD_SERVICE_HOST') and os.getenv(
                    'PADDLE_ELASTIC_ETCD_SERVICE_PORT'):
            server = '{}:{}'.format(
                os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_HOST'),
                os.getenv('PADDLE_ELASTIC_ETCD_SERVICE_PORT'))

        #elastic_timeout = os.getenv('PADDLE_ELASTIC_TIMEOUT',1)

        logger.debug('init with server {} host {}'.format(server, host))

        self.hosts = []
        self.stopped = False

        self.sigint = 0
        self.need_sync = False

        if not server or ':' not in server or not name or not np:
            logger.info(
                'Elastic is not enabled with server {} name {} and np {}'.
                format(server, name, np))
            self.enable = False
            return
        else:
            self.enable = True

        import etcd3

        srv, port = server.split(':')
        self.etcd = etcd3.client(host=srv, port=port)
        self.host = host if host else self._get_host()

        # etcd data
        self.prefix = "/paddle/" + name
        self.node_prefix = self.prefix + '/nodes'
        self.np_path = self.prefix + '/np'
        self.endpoints_path = self.prefix + '/endpoints'

        node_tag = ''.join(
            random.choice('abcdefghijklmnopqrstuvwxyz') for _ in range(6))
        self.host_path = '{}/{}{}'.format(self.node_prefix, node_tag,
                                          time.time())

        self.np = np + scale
        '''
        0 group mode, be aware of healthy status of other workers
        1 decouple mode, check own status only
        '''
        self.etcd.put(self.prefix, b'0')

        # host
        # register self host to etcd
        # register watch to reset host after host been deleted
        self.etcd.delete_prefix(self.node_prefix)

        def host_call_back(event):
            if self.etcd.get(self.host_path)[0] == None:
                logger.info('register host again {}'.format(self.host))

                self.etcd.put(self.host_path, six.b(self.host))
                self.need_sync = True

        host_watch = self.etcd.add_watch_callback(self.host_path,
                                                  host_call_back)
        self.etcd.put(self.host_path, six.b(self.host))

        # np describes the exact number of nodes to run the job
        inp = int(self.etcd.get(self.np_path)[0] or 0)
        if scale == 0 and not force:
            assert inp == np or inp == 0, "np {} is not consistent with np in etcd {}".format(
                np, inp)
        else:
            assert inp == np or inp == self.np, "np {} scale to {} by {} is not allowed".format(
                inp, self.np, scale)

        self.etcd.put(self.np_path, six.b("%d" % (self.np)))

        def np_call_back(event):
            gnp = int(self.etcd.get(self.np_path)[0])
            if gnp != self.np:
                logger.info("scale np {} to {} ".format(self.np, gnp))
                self.np = gnp

        np_watch = self.etcd.add_watch_callback(self.np_path, np_call_back)

        # endpoints handle DISTRIBUTED_TRAINER_ENDPOINTS and PADDLE_TRAINERS
        self.etcd.put(self.endpoints_path,
                      six.b('{}|{}'.format(self.endpoints, self.trainers)))

        def endpoints_call_back(event):
            if not self.endpoints:
                return
            edps = six.ensure_str(self.etcd.get(self.endpoints_path)[0] or '')
            self.endpoints, self.trainers = edps.split('|')
            logger.info("set DISTRIBUTED_TRAINER_ENDPOINTS {} ".format(
                self.endpoints))
            logger.info("set PADDLE_TRAINERS {} ".format(self.trainers))

        endpoints_watch = self.etcd.add_watch_callback(self.endpoints_path,
                                                       endpoints_call_back)

        self.watches = [host_watch, np_watch, endpoints_watch]

        self.launcher = None

    def exit(self, completed=False):
        logger.info('manager exist completed {}'.format(completed))

        if self.launcher:
            self.launcher.stop()

        if not self.enable:
            return

        if completed:
            self.etcd.put(self.prefix, b'1')

        for watch in self.watches:
            self.etcd.cancel_watch(watch)
        self.etcd.delete(self.host_path)

        hosts = [i for i in self.etcd.get_prefix(self.node_prefix)]
        if len(hosts) == 0:
            self.etcd.delete_prefix(self.prefix)

    def _get_host(self):
        try:
            return socket.gethostbyname(socket.getfqdn(socket.gethostname()))
        except:
            return '127.0.0.1'

    def _completed(self):
        if not self.enable:
            return True

        return int(self.etcd.get(self.prefix)[0]) == 1

    def _match(self):

        self.hosts = [
            six.ensure_str(i[0]) for i in self.etcd.get_prefix(self.node_prefix)
        ]
        if len(self.hosts) == self.np:
            return True
        else:
            return False

    def _update_hosts(self):
        assert len(self.hosts) != 0, 'hosts empty'

        if self.host in self.endpoints:
            os.environ['DISTRIBUTED_TRAINER_ENDPOINTS'] = self.endpoints
            os.environ['PADDLE_TRAINERS'] = self.trainers
            logger.info("update env DISTRIBUTED_TRAINER_ENDPOINTS {} ".format(
                self.endpoints))
            logger.info("update env PADDLE_TRAINERS {} ".format(self.trainers))
            return

        rank = int(os.getenv('PADDLE_TRAINER_ID', -1))
        idx = self.hosts.index(self.host)

        # swap if self.host not in the right position
        if rank >= 0:
            self.hosts[idx] = self.hosts[rank]
            self.hosts[rank] = self.host
        else:
            os.environ['PADDLE_TRAINER_ID'] = '{}'.format(idx)

        hosts = ','.join(self.hosts)
        self.args.ips = hosts
        os.environ['PADDLE_TRAINERS'] = hosts

    def wait(self):
        if not self.enable:
            return

        idx = 1
        while not self.stopped:
            if self._match():
                logger.info('ready with hosts {}'.format(self.hosts))
                self._update_hosts()
                return
            logger.info('not ready for np {} with hosts {}'.format(self.np,
                                                                   self.hosts))

            # reset hosts every 30s to prevent fake deadlock
            if idx % 10 == 0:
                self.etcd.delete_prefix(self.node_prefix)
                logger.info('reset np {} with hosts {}'.format(self.np,
                                                               self.hosts))

            idx += 1
            time.sleep(2)

        return

    def run(self, launcher):
        if self.stopped:
            return

        self.launcher = launcher(self.args)
        self.launcher.launch()

    def watch(self):

        if self.need_sync:
            self.need_sync = False

        while not self.stopped:
            ret = self.launcher.watch()

            if ret is not None:  # self terminated
                logger.info('job exit with code {}'.format(ret))
                # process is completed if ret >= 0 or error else
                completed = True if ret == 0 else False
                self.exit(completed=completed)
                if completed:
                    return ElasticStatus.COMPLETED
                if self.elastic_level == 1:
                    return ElasticStatus.RESTART
                else:
                    return ElasticStatus.ERROR

            if not self._completed() and (not self._match() or self.need_sync):
                self.launcher.stop()
                return ElasticStatus.HOLD

            time.sleep(2)

        if self.launcher:
            self.launcher.stop()
        return ElasticStatus.EXIT

    def signal_handler(self, sigint, frame):
        if self.enable:
            self.exit()
        self.sigint = sigint
        self.stopped = True
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import pdb

"""
Note, requests are much better than this
"""

import logging as mod_logging
import urllib2 as mod_urllib2
import urllib as mod_urllib
import collections as mod_collections

mod_logging.basicConfig(level=mod_logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')

VALID_METHODS = ['GET', 'DELETE', 'PUT', 'POST']

# For those methods params will be urlencoded and put in request body:
PARAMS_IN_BODY_METHODS = ['PUT', 'POST']

CustomHttpResponse = mod_collections.namedtuple(
        'CustomHttpResponse',
        ('status_code', 'headers', 'content'))

class CustomRequest(mod_urllib2.Request):

    def __init__(self, method, url, *args1, **args2):
        mod_urllib2.Request.__init__(self, url, *args1, **args2)
        self.method = method

    def get_method(self):
        return self.method

def add_params(url, urlencoded_params):
    if '?' in url:
        return url + '&' + urlencoded_params
    else:
        return url + '?' + urlencoded_params

def parse_headers(headers):
    result = {}
    if not headers:
        return result
    for header in headers:
        pos = header.find(':')
        if pos > 0:
            result[header[:pos]] = header[pos + 1:].strip()

    return result

def urlencode_params(params):
    if hasattr(params, 'items') or isinstance(params, tuple):
        return mod_urllib.urlencode(params)

    return params

def execute_request(method, url, data=None, headers=None):
    if not method in VALID_METHODS:
        raise Exception('Invalid method %s' % method)

    mod_logging.debug('%s to %s with %s', method, url, data)

    body = None
    opener = mod_urllib2.build_opener(mod_urllib2.HTTPHandler)
    if data:
        if method in PARAMS_IN_BODY_METHODS:
            body = urlencode_params(data)
        else:
            url = add_params(url, urlencode_params(data))

    request = CustomRequest(method, url)

    if headers:
        mod_logging.debug('Headers: %s', headers)
        for key, value in headers.items():
            request.add_header(key, value)

    try:
        url = opener.open(request, data=body)

        http_code = url.getcode()
        headers = headers
        body = url.read()
    except Exception, e:
        if hasattr(e, 'code'):
            http_code = e.code
        else:
            # Withoud http code => non-http exception:
            raise e

        if hasattr(e, 'headers'):
            headers = {}
            for key, value in e.headers.items():
                headers[key] = value

        if hasattr(e, 'read'):
            body = e.read()
    finally:
        try:
            url.close()
        except Exception, ignore:
            pass

    mod_logging.debug('http response code:%s', http_code)
    mod_logging.debug('http response headers:%s', headers)
    mod_logging.debug('http response body:%s', body)

    return CustomHttpResponse(http_code, headers, body)

def execute_GET(url, data=None, headers=None):
    return execute_request('GET', url, data=data, headers=headers)

def execute_POST(url, data=None, headers=None):
    return execute_request('POST', url, data=data, headers=headers)

def execute_PUT(url, data=None, headers=None):
    return execute_request('PUT', url, data=data, headers=headers)

def execute_DELETE(url, data=None, headers=None):
    return execute_request('DELETE', url, data=data, headers=headers)

if __name__ == '__main__':
    #print execute_GET('http://localhost', {'a': 'šđčćž/(/(/('})
    print execute_GET('https://oneapi.infobip.com/status', {'a': 'šđčćž/(/(/('})
<commit_msg>Add support for JSON in execute_request HTTP python class<commit_after># -*- coding: utf-8 -*-

import pdb

"""
Note, requests are much better than this
"""
import json as mod_json
import logging as mod_logging
import urllib2 as mod_urllib2
import urllib as mod_urllib
import collections as mod_collections

mod_logging.basicConfig(level=mod_logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')

VALID_METHODS = ['GET', 'DELETE', 'PUT', 'POST']

# For those methods params will be urlencoded and put in request body:
PARAMS_IN_BODY_METHODS = ['PUT', 'POST']

CustomHttpResponse = mod_collections.namedtuple(
        'CustomHttpResponse',
        ('status_code', 'headers', 'content'))

class CustomRequest(mod_urllib2.Request):

    def __init__(self, method, url, *args1, **args2):
        mod_urllib2.Request.__init__(self, url, *args1, **args2)
        self.method = method

    def get_method(self):
        return self.method

def add_params(url, urlencoded_params):
    if '?' in url:
        return url + '&' + urlencoded_params
    else:
        return url + '?' + urlencoded_params

def parse_headers(headers):
    result = {}
    if not headers:
        return result
    for header in headers:
        pos = header.find(':')
        if pos > 0:
            result[header[:pos]] = header[pos + 1:].strip()

    return result

def urlencode_params(params):
    if hasattr(params, 'items') or isinstance(params, tuple):
        return mod_urllib.urlencode(params)

    return params

def execute_request(method, url, data=None, headers=None, data_format=None):
    if not method in VALID_METHODS:
        raise Exception('Invalid method %s' % method)

    mod_logging.debug('%s to %s with %s', method, url, data)

    body = None
    opener = mod_urllib2.build_opener(mod_urllib2.HTTPHandler)
    if data:
        if method in PARAMS_IN_BODY_METHODS:
            if data_format == "json":
                body = mod_json.JSONEncoder().encode(data)
            else:
                body = urlencode_params(data)
        else:
            url = add_params(url, urlencode_params(data))

    request = CustomRequest(method, url)

    if data_format == "json":
        headers["content-type"] = "application/json"

    if headers:
        mod_logging.debug('Headers: %s', headers)
        for key, value in headers.items():
            request.add_header(key, value)

    try:
        url = opener.open(request, data=body)

        http_code = url.getcode()
        headers = headers
        body = url.read()
    except Exception, e:
        if hasattr(e, 'code'):
            http_code = e.code
        else:
            # Withoud http code => non-http exception:
            raise e

        if hasattr(e, 'headers'):
            headers = {}
            for key, value in e.headers.items():
                headers[key] = value

        if hasattr(e, 'read'):
            body = e.read()
    finally:
        try:
            url.close()
        except Exception, ignore:
            pass

    mod_logging.debug('http response code:%s', http_code)
    mod_logging.debug('http response headers:%s', headers)
    mod_logging.debug('http response body:%s', body)

    return CustomHttpResponse(http_code, headers, body)

def execute_GET(url, data=None, headers=None):
    return execute_request('GET', url, data=data, headers=headers)

def execute_POST(url, data=None, headers=None):
    return execute_request('POST', url, data=data, headers=headers)

def execute_PUT(url, data=None, headers=None):
    return execute_request('PUT', url, data=data, headers=headers)

def execute_DELETE(url, data=None, headers=None):
    return execute_request('DELETE', url, data=data, headers=headers)

if __name__ == '__main__':
    #print execute_GET('http://localhost', {'a': 'šđčćž/(/(/('})
    print execute_GET('https://oneapi.infobip.com/status', {'a': 'šđčćž/(/(/('})
<|endoftext|>


<commit_before># Copyright (c) 2016, the Cap authors.
#
# This file is subject to the Modified BSD License and may not be distributed
# without copyright and license information. Please refer to the file LICENSE
# for the text and further information on this license.

from pycap import PropertyTree
import unittest

class boostPropertyTreePythonWrappersTestCase(unittest.TestCase):
    def test_pickle_support(self):
        import pickle
        src = PropertyTree()
        src.put_double('pi', 3.14)
        src.put_string('greet', 'bonjour')
        p = pickle.dumps(src)
        dst = pickle.loads(p)
        self.assertEqual(dst.get_double('pi'), 3.14)
        self.assertEqual(dst.get_string('greet'), 'bonjour')

    def test_get_with_default_value(self):
        ptree = PropertyTree()
        # double
        self.assertEqual(ptree.get_double_with_default_value('missing_double',
                                                             3.14), 3.14)
        ptree.put_double('present_double', 1.41)
        self.assertEqual(ptree.get_double_with_default_value('present_double',
                                                             3.14), 1.41)
        # string
        self.assertEqual(ptree.get_string_with_default_value('missing_string',
                                                             'missing'),
                         'missing')
        ptree.put_string('present_string', 'present')
        self.assertEqual(ptree.get_string_with_default_value('present_string',
                                                             'missing'),
                         'present')
        # int
        self.assertEqual(ptree.get_int_with_default_value('missing_int', 255),
                         255)
        ptree.put_int('present_int', 0)
        self.assertEqual(ptree.get_int_with_default_value('present_int', 255),
                         0)
        # bool
        self.assertEqual(ptree.get_bool_with_default_value('missing_bool',
                                                           True), True)
        ptree.put_bool('present_bool', False)
        self.assertEqual(ptree.get_bool_with_default_value('present_bool',
                                                           True), False)

    def test_get_array(self):
        ptree = PropertyTree()
        # array of double
        ptree.put_string('array_double', '3.14,1.41')
        array_double = ptree.get_array_double('array_double')
        self.assertEqual(array_double, [3.14, 1.41])
        # ... string
        ptree.put_string('array_int', '1,2,3')
        array_int = ptree.get_array_int('array_int')
        self.assertEqual(array_int, [1, 2, 3])
        # ... int
        ptree.put_string('array_string', 'uno,dos,tres,cuatro')
        array_string = ptree.get_array_string('array_string')
        self.assertEqual(array_string, ['uno', 'dos', 'tres', 'cuatro'])
        # ... bool
        ptree.put_string('array_bool', 'true,FALSE,False')
        array_bool = ptree.get_array_bool('array_bool')
        self.assertEqual(array_bool, [True, False, False])

    def test_property_tree(self):
        # ptree as container to store int, double, string, and bool
        ptree = PropertyTree()
        ptree.put_int('dim', 3)
        self.assertEqual(ptree.get_int('dim'), 3)
        ptree.put_double('path.to.pi', 3.14)
        self.assertEqual(ptree.get_double('path.to.pi'), 3.14)
        ptree.put_string('good.news', 'it works')
        self.assertEqual(ptree.get_string('good.news'), 'it works')
        ptree.put_bool('is.that.a.good.idea', False)
        self.assertEqual(ptree.get_bool('is.that.a.good.idea'), False)

    def test_get_child(self):
        ptree = PropertyTree()
        ptree.put_string('child.name', 'clement')
        ptree.put_int('child.age', -2)
        child = ptree.get_child('child')
        self.assertEqual(child.get_string('name'), 'clement')
        self.assertEqual(child.get_int('age'), -2)

    def test_raise_exceptions(self):
        ptree = PropertyTree()
        # property tree will throw if the specified path does not exist
        self.assertRaises(RuntimeError, ptree.get_int, 'path.does.not.exist')
        # or if the translation fails
        ptree.put_string('some.path.to.a.string', 'not a double')
        self.assertRaises(RuntimeError, ptree.get_double,
                          'some.path.to.a.string')

    # TODO
    def test_parse(self):
        ptree = PropertyTree()
        ptree.parse_xml('device.xml')

if __name__ == '__main__':
    unittest.main()
<commit_msg>test ptree parsers in python<commit_after># Copyright (c) 2016, the Cap authors.
#
# This file is subject to the Modified BSD License and may not be distributed
# without copyright and license information. Please refer to the file LICENSE
# for the text and further information on this license.

from pycap import PropertyTree
from os import remove
import unittest

class boostPropertyTreePythonWrappersTestCase(unittest.TestCase):
    def test_pickle_support(self):
        import pickle
        src = PropertyTree()
        src.put_double('pi', 3.14)
        src.put_string('greet', 'bonjour')
        p = pickle.dumps(src)
        dst = pickle.loads(p)
        self.assertEqual(dst.get_double('pi'), 3.14)
        self.assertEqual(dst.get_string('greet'), 'bonjour')

    def test_get_with_default_value(self):
        ptree = PropertyTree()
        # double
        self.assertEqual(ptree.get_double_with_default_value('missing_double',
                                                             3.14), 3.14)
        ptree.put_double('present_double', 1.41)
        self.assertEqual(ptree.get_double_with_default_value('present_double',
                                                             3.14), 1.41)
        # string
        self.assertEqual(ptree.get_string_with_default_value('missing_string',
                                                             'missing'),
                         'missing')
        ptree.put_string('present_string', 'present')
        self.assertEqual(ptree.get_string_with_default_value('present_string',
                                                             'missing'),
                         'present')
        # int
        self.assertEqual(ptree.get_int_with_default_value('missing_int', 255),
                         255)
        ptree.put_int('present_int', 0)
        self.assertEqual(ptree.get_int_with_default_value('present_int', 255),
                         0)
        # bool
        self.assertEqual(ptree.get_bool_with_default_value('missing_bool',
                                                           True), True)
        ptree.put_bool('present_bool', False)
        self.assertEqual(ptree.get_bool_with_default_value('present_bool',
                                                           True), False)

    def test_get_array(self):
        ptree = PropertyTree()
        # array of double
        ptree.put_string('array_double', '3.14,1.41')
        array_double = ptree.get_array_double('array_double')
        self.assertEqual(array_double, [3.14, 1.41])
        # ... string
        ptree.put_string('array_int', '1,2,3')
        array_int = ptree.get_array_int('array_int')
        self.assertEqual(array_int, [1, 2, 3])
        # ... int
        ptree.put_string('array_string', 'uno,dos,tres,cuatro')
        array_string = ptree.get_array_string('array_string')
        self.assertEqual(array_string, ['uno', 'dos', 'tres', 'cuatro'])
        # ... bool
        ptree.put_string('array_bool', 'true,FALSE,False')
        array_bool = ptree.get_array_bool('array_bool')
        self.assertEqual(array_bool, [True, False, False])

    def test_property_tree(self):
        # ptree as container to store int, double, string, and bool
        ptree = PropertyTree()
        ptree.put_int('dim', 3)
        self.assertEqual(ptree.get_int('dim'), 3)
        ptree.put_double('path.to.pi', 3.14)
        self.assertEqual(ptree.get_double('path.to.pi'), 3.14)
        ptree.put_string('good.news', 'it works')
        self.assertEqual(ptree.get_string('good.news'), 'it works')
        ptree.put_bool('is.that.a.good.idea', False)
        self.assertEqual(ptree.get_bool('is.that.a.good.idea'), False)

    def test_get_child(self):
        ptree = PropertyTree()
        ptree.put_string('child.name', 'clement')
        ptree.put_int('child.age', -2)
        child = ptree.get_child('child')
        self.assertEqual(child.get_string('name'), 'clement')
        self.assertEqual(child.get_int('age'), -2)

    def test_raise_exceptions(self):
        ptree = PropertyTree()
        # property tree will throw if the specified path does not exist
        self.assertRaises(RuntimeError, ptree.get_int, 'path.does.not.exist')
        # or if the translation fails
        ptree.put_string('some.path.to.a.string', 'not a double')
        self.assertRaises(RuntimeError, ptree.get_double,
                          'some.path.to.a.string')

    def test_parsers(self):
        # populate a populate property tree from an input file
        ptree = PropertyTree()
        # INFO parser
        info_file = 'input.info'
        today = 'april 8th, 2016'
        with open(info_file, 'w') as fout:
            fout.write('date "'+today+'"')
        ptree.parse_info(info_file)
        self.assertEqual(ptree.get_string('date'), today)
        remove(info_file)
        # XML parser
        xml_file = 'input.xml'
        pi = 3.14
        with open(xml_file, 'w') as fout:
            fout.write('<pi>'+str(pi)+'</pi>')
        ptree.parse_xml(xml_file)
        self.assertEqual(ptree.get_double('pi'), pi)
        remove(xml_file)
        # JSON parser
        json_file = 'input.json'
        with open(json_file, 'w') as fout:
            fout.write('{')
            fout.write('  "foo":')
            fout.write('  {')
            fout.write('    "bar": false')
            fout.write('  }')
            fout.write('}')
        ptree.parse_json(json_file)
        self.assertFalse(ptree.get_bool('foo.bar'))
        remove(json_file)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from pywinauto.application import Application
from pywinauto.handleprops import processid
from pywinauto.sysinfo import is_x64_Python, is_x64_OS

from pywinauto.UIAElementInfo import UIAElementInfo

import unittest

class UIAElementInfoTests(unittest.TestCase):
    "Unit tests for the UIElementInfo class"

    def setUp(self):
        """Start the application set some data and ensure the application
        is in the state we want it."""
        # start the application
        self.app = Application()
        if is_x64_Python() or not is_x64_OS():
            self.app.start_(r"C:\Windows\System32\calc.exe")
        else:
            self.app.start_(r"C:\Windows\SysWOW64\calc.exe")

        self.dlg = self.app.Calculator
        self.handle = self.dlg.handle
        self.dlg.MenuSelect('View->Scientific\tAlt+2')
        self.ctrl = UIAElementInfo(self.dlg.handle)

    def tearDown(self):
        "Close the application after tests"
        self.app.kill_()

    def testProcessId(self):
        self.assertEqual(self.ctrl.processId, processid(self.handle))

    def testName(self):
        self.assertEqual(self.ctrl.name, "Calculator")

    def testHandle(self):
        self.assertEqual(self.ctrl.handle, self.handle)

    def testSetHandle(self):
        element = UIAElementInfo(self.dlg.Button2.handle)
        element.handle = self.handle
        self.assertEqual(self.ctrl.handle, self.handle)

    def testEnabled(self):
        self.assertEqual(self.ctrl.enabled, True)

    def testVisible(self):
        self.assertEqual(self.ctrl.visible, True)

    def testChildren(self):
        self.assertEqual(len(self.ctrl.children), 3)

if __name__ == "__main__":
    unittest.main()
<commit_msg>Disable UIA tests in native-only mode.<commit_after>from pywinauto.application import Application
from pywinauto.handleprops import processid
from pywinauto.sysinfo import is_x64_Python, is_x64_OS, UIA_support

if UIA_support:
    from pywinauto.UIAElementInfo import UIAElementInfo

import unittest

if UIA_support:
    class UIAElementInfoTests(unittest.TestCase):
        "Unit tests for the UIElementInfo class"

        def setUp(self):
            """Start the application set some data and ensure the application
            is in the state we want it."""
            # start the application
            self.app = Application()
            if is_x64_Python() or not is_x64_OS():
                self.app.start_(r"C:\Windows\System32\calc.exe")
            else:
                self.app.start_(r"C:\Windows\SysWOW64\calc.exe")

            self.dlg = self.app.Calculator
            self.handle = self.dlg.handle
            self.dlg.MenuSelect('View->Scientific\tAlt+2')
            self.ctrl = UIAElementInfo(self.dlg.handle)

        def tearDown(self):
            "Close the application after tests"
            self.app.kill_()

        def testProcessId(self):
            self.assertEqual(self.ctrl.processId, processid(self.handle))

        def testName(self):
            self.assertEqual(self.ctrl.name, "Calculator")

        def testHandle(self):
            self.assertEqual(self.ctrl.handle, self.handle)

        def testSetHandle(self):
            element = UIAElementInfo(self.dlg.Button2.handle)
            element.handle = self.handle
            self.assertEqual(self.ctrl.handle, self.handle)

        def testEnabled(self):
            self.assertEqual(self.ctrl.enabled, True)

        def testVisible(self):
            self.assertEqual(self.ctrl.visible, True)

        def testChildren(self):
            self.assertEqual(len(self.ctrl.children), 3)

if __name__ == "__main__":
    if UIA_support:
        unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Sample script of word embedding model.

This code implements skip-gram model and continuous-bow model.
Use ../ptb/download.py to download 'ptb.train.txt'.
"""
import argparse
import collections
import time

import numpy as np
import six.moves.cPickle as pickle

import chainer
from chainer import cuda
import chainer.functions as F
import chainer.optimizers as O

parser = argparse.ArgumentParser()
parser.add_argument('--gpu', '-g', default=-1, type=int,
                    help='GPU ID (negative value indicates CPU)')
parser.add_argument('--unit', '-u', default=100, type=int,
                    help='number of units')
parser.add_argument('--window', '-w', default=5, type=int,
                    help='window size')
parser.add_argument('--batchsize', '-b', type=int, default=100,
                    help='learning minibatch size')
parser.add_argument('--epoch', '-e', default=10, type=int,
                    help='number of epochs to learn')
parser.add_argument('--model', '-m', choices=['skipgram', 'cbow'],
                    default='skipgram',
                    help='model type ("skipgram", "cbow")')
parser.add_argument('--out-type', '-o', choices=['hsm', 'ns', 'original'],
                    default='hsm',
                    help='output model type ("hsm": hierarchical softmax, '
                    '"ns": negative sampling, "original": no approximation)')
args = parser.parse_args()

print('GPU: {}'.format(args.gpu))
print('# unit: {}'.format(args.unit))
print('Window: {}'.format(args.window))
print('Minibatch-size: {}'.format(args.batchsize))
print('# epoch: {}'.format(args.epoch))
print('Training model: {}'.format(args.model))
print('Output type: {}'.format(args.out_type))
print('')

def continuous_bow(dataset, position):
    h = None

    # use random window size in the same way as the original word2vec
    # implementation.
    w = np.random.randint(args.window - 1) + 1
    for offset in range(-w, w + 1):
        if offset == 0:
            continue
        d = dataset[position + offset]
        if args.gpu >= 0:
            d = cuda.to_gpu(d)
        x = chainer.Variable(d)
        e = model.embed(x)
        h = h + e if h is not None else e

    d = dataset[position]
    if args.gpu >= 0:
        d = cuda.to_gpu(d)
    t = chainer.Variable(d)
    return loss_func(h, t)

def skip_gram(dataset, position):
    d = dataset[position]
    if args.gpu >= 0:
        d = cuda.to_gpu(d)
    t = chainer.Variable(d)

    # use random window size in the same way as the original word2vec
    # implementation.
    w = np.random.randint(args.window - 1) + 1
    loss = None
    for offset in range(-w, w + 1):
        if offset == 0:
            continue
        d = dataset[position + offset]
        if args.gpu >= 0:
            d = cuda.to_gpu(d)
        x = chainer.Variable(d)
        e = model.embed(x)

        loss_i = loss_func(e, t)
        loss = loss_i if loss is None else loss + loss_i

    return loss

if args.gpu >= 0:
    cuda.init(args.gpu)

index2word = {}
word2index = {}
counts = collections.defaultdict(lambda: 0)
dataset = []
with open('ptb.train.txt') as f:
    for line in f:
        for word in line.split():
            if word not in word2index:
                ind = len(word2index)
                word2index[word] = ind
                index2word[ind] = word
            counts[word2index[word]] += 1
            dataset.append(word2index[word])

n_vocab = len(word2index)

print('n_vocab: %d' % n_vocab)
print('data length: %d' % len(dataset))

if args.model == 'skipgram':
    train_model = skip_gram
elif args.model == 'cbow':
    train_model = continuous_bow
else:
    raise Exception('Unknown model type: {}'.format(args.model))

model = chainer.FunctionSet(
    embed=F.EmbedID(n_vocab, args.unit),
)

if args.out_type == 'hsm':
    tree = F.create_huffman_tree(counts)
    model.l = F.BinaryHierarchicalSoftmax(args.unit, tree)
    loss_func = model.l
elif args.out_type == 'ns':
    cs = [counts[w] for w in range(len(counts))]
    model.l = F.NegativeSampling(args.unit, cs, 20)
    loss_func = model.l
elif args.out_type == 'original':
    model.l = F.Linear(args.unit, n_vocab)
    loss_func = lambda h, t: F.softmax_cross_entropy(model.l(h), t)
else:
    raise Exception('Unknown output type: {}'.format(args.out_type))

if args.gpu >= 0:
    model.to_gpu()

dataset = np.array(dataset, dtype=np.int32)

optimizer = O.Adam()
optimizer.setup(model.collect_parameters())

begin_time = time.time()
cur_at = begin_time
word_count = 0
skip = (len(dataset) - args.window * 2) // args.batchsize
next_count = 100000
for epoch in range(args.epoch):
    accum_loss = 0
    print('epoch: {0}'.format(epoch))
    indexes = np.random.permutation(skip)
    for i in indexes:
        if word_count >= next_count:
            now = time.time()
            duration = now - cur_at
            throuput = 100000. / (now - cur_at)
            print('{} words, {:.2f} sec, {:.2f} words/sec'.format(
                word_count, duration, throuput))
            next_count += 100000
            cur_at = now

        position = np.array(
            range(0, args.batchsize)) * skip + (args.window + i)
        loss = train_model(dataset, position)
        accum_loss += loss.data
        word_count += args.batchsize

        optimizer.zero_grads()
        loss.backward()
        optimizer.update()

    print(accum_loss)

model.to_cpu()
with open('model.pickle', 'wb') as f:
    obj = (model, index2word, word2index)
    pickle.dump(obj, f)
<commit_msg>Use Counter instead of defaultdict<commit_after>#!/usr/bin/env python
"""Sample script of word embedding model.

This code implements skip-gram model and continuous-bow model.
Use ../ptb/download.py to download 'ptb.train.txt'.
"""
import argparse
import collections
import time

import numpy as np
import six.moves.cPickle as pickle

import chainer
from chainer import cuda
import chainer.functions as F
import chainer.optimizers as O

parser = argparse.ArgumentParser()
parser.add_argument('--gpu', '-g', default=-1, type=int,
                    help='GPU ID (negative value indicates CPU)')
parser.add_argument('--unit', '-u', default=100, type=int,
                    help='number of units')
parser.add_argument('--window', '-w', default=5, type=int,
                    help='window size')
parser.add_argument('--batchsize', '-b', type=int, default=100,
                    help='learning minibatch size')
parser.add_argument('--epoch', '-e', default=10, type=int,
                    help='number of epochs to learn')
parser.add_argument('--model', '-m', choices=['skipgram', 'cbow'],
                    default='skipgram',
                    help='model type ("skipgram", "cbow")')
parser.add_argument('--out-type', '-o', choices=['hsm', 'ns', 'original'],
                    default='hsm',
                    help='output model type ("hsm": hierarchical softmax, '
                    '"ns": negative sampling, "original": no approximation)')
args = parser.parse_args()

print('GPU: {}'.format(args.gpu))
print('# unit: {}'.format(args.unit))
print('Window: {}'.format(args.window))
print('Minibatch-size: {}'.format(args.batchsize))
print('# epoch: {}'.format(args.epoch))
print('Training model: {}'.format(args.model))
print('Output type: {}'.format(args.out_type))
print('')

def continuous_bow(dataset, position):
    h = None

    # use random window size in the same way as the original word2vec
    # implementation.
    w = np.random.randint(args.window - 1) + 1
    for offset in range(-w, w + 1):
        if offset == 0:
            continue
        d = dataset[position + offset]
        if args.gpu >= 0:
            d = cuda.to_gpu(d)
        x = chainer.Variable(d)
        e = model.embed(x)
        h = h + e if h is not None else e

    d = dataset[position]
    if args.gpu >= 0:
        d = cuda.to_gpu(d)
    t = chainer.Variable(d)
    return loss_func(h, t)

def skip_gram(dataset, position):
    d = dataset[position]
    if args.gpu >= 0:
        d = cuda.to_gpu(d)
    t = chainer.Variable(d)

    # use random window size in the same way as the original word2vec
    # implementation.
    w = np.random.randint(args.window - 1) + 1
    loss = None
    for offset in range(-w, w + 1):
        if offset == 0:
            continue
        d = dataset[position + offset]
        if args.gpu >= 0:
            d = cuda.to_gpu(d)
        x = chainer.Variable(d)
        e = model.embed(x)

        loss_i = loss_func(e, t)
        loss = loss_i if loss is None else loss + loss_i

    return loss

if args.gpu >= 0:
    cuda.init(args.gpu)

index2word = {}
word2index = {}
counts = collections.Counter()
dataset = []
with open('ptb.train.txt') as f:
    for line in f:
        for word in line.split():
            if word not in word2index:
                ind = len(word2index)
                word2index[word] = ind
                index2word[ind] = word
            counts[word2index[word]] += 1
            dataset.append(word2index[word])

n_vocab = len(word2index)

print('n_vocab: %d' % n_vocab)
print('data length: %d' % len(dataset))

if args.model == 'skipgram':
    train_model = skip_gram
elif args.model == 'cbow':
    train_model = continuous_bow
else:
    raise Exception('Unknown model type: {}'.format(args.model))

model = chainer.FunctionSet(
    embed=F.EmbedID(n_vocab, args.unit),
)

if args.out_type == 'hsm':
    tree = F.create_huffman_tree(counts)
    model.l = F.BinaryHierarchicalSoftmax(args.unit, tree)
    loss_func = model.l
elif args.out_type == 'ns':
    cs = [counts[w] for w in range(len(counts))]
    model.l = F.NegativeSampling(args.unit, cs, 20)
    loss_func = model.l
elif args.out_type == 'original':
    model.l = F.Linear(args.unit, n_vocab)
    loss_func = lambda h, t: F.softmax_cross_entropy(model.l(h), t)
else:
    raise Exception('Unknown output type: {}'.format(args.out_type))

if args.gpu >= 0:
    model.to_gpu()

dataset = np.array(dataset, dtype=np.int32)

optimizer = O.Adam()
optimizer.setup(model.collect_parameters())

begin_time = time.time()
cur_at = begin_time
word_count = 0
skip = (len(dataset) - args.window * 2) // args.batchsize
next_count = 100000
for epoch in range(args.epoch):
    accum_loss = 0
    print('epoch: {0}'.format(epoch))
    indexes = np.random.permutation(skip)
    for i in indexes:
        if word_count >= next_count:
            now = time.time()
            duration = now - cur_at
            throuput = 100000. / (now - cur_at)
            print('{} words, {:.2f} sec, {:.2f} words/sec'.format(
                word_count, duration, throuput))
            next_count += 100000
            cur_at = now

        position = np.array(
            range(0, args.batchsize)) * skip + (args.window + i)
        loss = train_model(dataset, position)
        accum_loss += loss.data
        word_count += args.batchsize

        optimizer.zero_grads()
        loss.backward()
        optimizer.update()

    print(accum_loss)

model.to_cpu()
with open('model.pickle', 'wb') as f:
    obj = (model, index2word, word2index)
    pickle.dump(obj, f)
<|endoftext|>


<commit_before>from django.shortcuts import render
from order.models import *
from member.models import University
from order.forms import CATEGORY, ORIGIN
from django.contrib.auth.decorators import login_required, permission_required
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
import json as simplejson
import json
import datetime

# Create your views here.

@login_required
def show_institute(request):
    if request.method == 'GET':
        university_id = request.GET.get('university')
        university = get_object_or_404(University, id=university_id)

        select = Institute.objects.filter(university=university)
        institute = []
        for inst in select:
            institute.append({'pk': inst.id, 'valor': inst.__unicode__()})

        json = simplejson.dumps(institute)
        return HttpResponse(json, content_type="application/json")

@login_required
def show_department(request):
    if request.method == 'GET':
        institute_id = request.GET.get('institute')
        institute = get_object_or_404(Institute, id=institute_id)

        select = Department.objects.filter(institute=institute)
        department = []
        for dep in select:
            department.append({'pk': dep.id, 'valor': dep.__unicode__()})

        json = simplejson.dumps(department)
        return HttpResponse(json, content_type="application/json")

@login_required
#@permission_required('is_superuser', raise_exception=True)
def scientific_missions_report(request):

    if request.method == 'POST':
        time = " 00:00:00"
        start_date = request.POST['start_date']
        if start_date:
            start_day = start_date[0:2]
            start_month = start_date[3:5]
            start_year = start_date[6:10]
            start_date = start_year+start_month+start_day+time
            start_date = datetime.datetime.strptime(start_date, "%Y%m%d %H:%M:%S").date()
            start_date -= datetime.timedelta(days=1)
        else:
            start_date = datetime.datetime.strptime('19700101 00:00:00', '%Y%m%d %H:%M:%S').date()

        end_date = request.POST['end_date']
        if end_date:
            end_day = end_date[0:2]
            end_month = end_date[3:5]
            end_year = end_date[6:10]
            end_date = end_year+end_month+end_day+time
            end_date = datetime.datetime.strptime(end_date, "%Y%m%d %H:%M:%S").date()
            end_date += datetime.timedelta(days=1)
        else:
            now_plus_30 = datetime.datetime.now() + datetime.timedelta(days=30)
            now_plus_30 = now_plus_30.strftime("%Y%m%d %H:%M:%S")
            end_date = datetime.datetime.strptime(now_plus_30, '%Y%m%d %H:%M:%S').date()

        orders = Order.objects.filter(type_of_order='d', status='f', dailystipend__departure__gt=start_date,
                                      dailystipend__arrival__lt=end_date)

        context = {'orders': orders}
        return render(request, 'report/scientific_missions_report.html', context)

    return render(request, 'report/scientific_missions.html')

@login_required
def list_order_by_type(request):
    types = [{'value': order_type[0], 'display': order_type[1].encode('utf-8')} for order_type in ORDER_TYPE]

    if request.method == 'POST':

        if request.POST['order_type'] == "h":
            status = request.POST.get('status')
            category = request.POST.get('category')
            origin = request.POST.get('origin')
            if status == '' and category == '' and origin == '':
                orders = Order.objects.filter(type_of_order='h')
            elif status == '' and category == '':
                orders = Order.objects.filter(type_of_order='h', hardwaresoftware__origin=origin)
            elif status == '' and origin == '':
                orders = Order.objects.filter(type_of_order='h', hardwaresoftware__category=category)
            elif category == '' and origin == '':
                orders = Order.objects.filter(type_of_order='h', status=status)
            elif status == '':
                orders = Order.objects.filter(type_of_order='h', hardwaresoftware__origin=origin,
                                              hardwaresoftware__category=category)
            elif category == '':
                orders = Order.objects.filter(type_of_order='h', status=status, hardwaresoftware__origin=origin)
            elif origin == '':
                orders = Order.objects.filter(type_of_order='h', status=status, hardwaresoftware__category=category)
            else:
                orders = Order.objects.filter(type_of_order='h', status=status, hardwaresoftware__category=category,
                                              hardwaresoftware__origin=origin)

            context = {'orders': orders, 'status': status, 'category': category, 'origin': origin}
            return render(request, 'report/list_equipment_supplies_msc.html', context)

        elif request.POST['order_type'] == "s":
            status = request.POST.get('status')
            origin = request.POST.get('origin')
            if status == '' and origin == '':
                orders = Order.objects.filter(type_of_order='s')
            elif status == '':
                orders = Order.objects.filter(type_of_order='s', service__origin=origin)
            elif origin == '':
                orders = Order.objects.filter(type_of_order='s', status=status)
            else:
                orders = Order.objects.filter(type_of_order='s', status=status, service__origin=origin)

            context = {'orders': orders}
            return render(request, 'report/list_services.html', context)

        elif request.POST['order_type'] == "e":
            status = request.POST.get('status')
            if status == '':
                orders = Order.objects.filter(type_of_order='e')
            else:
                orders = Order.objects.filter(type_of_order='e', status=status)

            context = {'orders': orders}
            return render(request, 'report/list_events.html', context)

        elif request.POST['order_type'] == "t":
            status = request.POST.get('status')
            if status == '':
                orders = Order.objects.filter(type_of_order='t')
            else:
                orders = Order.objects.filter(type_of_order='t', status=status)

            context = {'orders': orders}
            return render(request, 'report/list_tickets.html', context)

        elif request.POST['order_type'] == "d":
            status = request.POST.get('status')
            if status == '':
                orders = Order.objects.filter(type_of_order='d')
            else:
                orders = Order.objects.filter(type_of_order='d', status=status)

            context = {'orders': orders}
            return render(request, 'report/list_daily_stipend.html', context)

        elif request.POST['order_type'] == "r":
            status = request.POST.get('status')
            if status == '':
                orders = Order.objects.filter(type_of_order='r')
            else:
                orders = Order.objects.filter(type_of_order='r', status=status)

            context = {'orders': orders}
            return render(request, 'report/list_reimbursement.html', context)

    context = {'types': types}
    return render(request, 'report/list_order.html', context)

@login_required
def select_additional_options(request):
    orders = request.GET.get('order_type')
    if orders == 'h':
        status = [{'value': status[0], 'display': status[1].encode('utf-8')} for status in ORDER_STATUS]
        categories = [{'value': category[0], 'display': category[1].encode('utf-8')} for category in CATEGORY]
        origin = [{'value': orig[0], 'display': orig[1].encode('utf-8')} for orig in ORIGIN]
    elif orders == 's':
        status = [{'value': status[0], 'display': status[1].encode('utf-8')} for status in ORDER_STATUS]
        origin = [{'value': orig[0], 'display': orig[1].encode('utf-8')} for orig in ORIGIN]
        categories = []
    else:
        status = [{'value': status[0], 'display': status[1].encode('utf-8')} for status in ORDER_STATUS]
        categories = []
        origin = []
    return HttpResponse(json.dumps([status, categories, origin]), content_type="application/json")<commit_msg>Shows a message if End date is smaller than Start date.<commit_after>from django.shortcuts import render
from order.models import *
from member.models import University
from order.forms import CATEGORY, ORIGIN
from django.contrib.auth.decorators import login_required, permission_required
from django.http import HttpResponse
from django.shortcuts import get_object_or_404
import json as simplejson
import json
import datetime
from django.contrib import messages

# Create your views here.

@login_required
def show_institute(request):
    if request.method == 'GET':
        university_id = request.GET.get('university')
        university = get_object_or_404(University, id=university_id)

        select = Institute.objects.filter(university=university)
        institute = []
        for inst in select:
            institute.append({'pk': inst.id, 'valor': inst.__unicode__()})

        json = simplejson.dumps(institute)
        return HttpResponse(json, content_type="application/json")

@login_required
def show_department(request):
    if request.method == 'GET':
        institute_id = request.GET.get('institute')
        institute = get_object_or_404(Institute, id=institute_id)

        select = Department.objects.filter(institute=institute)
        department = []
        for dep in select:
            department.append({'pk': dep.id, 'valor': dep.__unicode__()})

        json = simplejson.dumps(department)
        return HttpResponse(json, content_type="application/json")

@login_required
#@permission_required('is_superuser', raise_exception=True)
def scientific_missions_report(request):

    if request.method == 'POST':
        time = " 00:00:00"
        start_date = request.POST['start_date']
        if start_date:
            start_day = start_date[0:2]
            start_month = start_date[3:5]
            start_year = start_date[6:10]
            start_date = start_year+start_month+start_day+time
            start_date = datetime.datetime.strptime(start_date, "%Y%m%d %H:%M:%S").date()
            start_date -= datetime.timedelta(days=1)
        else:
            start_date = datetime.datetime.strptime('19700101 00:00:00', '%Y%m%d %H:%M:%S').date()

        end_date = request.POST['end_date']
        if end_date:
            end_day = end_date[0:2]
            end_month = end_date[3:5]
            end_year = end_date[6:10]
            end_date = end_year+end_month+end_day+time
            end_date = datetime.datetime.strptime(end_date, "%Y%m%d %H:%M:%S").date()
            end_date += datetime.timedelta(days=1)
        else:
            now_plus_30 = datetime.datetime.now() + datetime.timedelta(days=30)
            now_plus_30 = now_plus_30.strftime("%Y%m%d %H:%M:%S")
            end_date = datetime.datetime.strptime(now_plus_30, '%Y%m%d %H:%M:%S').date()

        orders = Order.objects.filter(type_of_order='d', status='f', dailystipend__departure__gt=start_date,
                                      dailystipend__arrival__lt=end_date)

        context = {'orders': orders}

        if end_date >= start_date:
            return render(request, 'report/scientific_missions_report.html', context)
        else:
            messages.error(request, 'End date should be equal or greater than Start date.')
            return render(request, 'report/scientific_missions.html')

    return render(request, 'report/scientific_missions.html')

@login_required
def list_order_by_type(request):
    types = [{'value': order_type[0], 'display': order_type[1].encode('utf-8')} for order_type in ORDER_TYPE]

    if request.method == 'POST':

        if request.POST['order_type'] == "h":
            status = request.POST.get('status')
            category = request.POST.get('category')
            origin = request.POST.get('origin')
            if status == '' and category == '' and origin == '':
                orders = Order.objects.filter(type_of_order='h')
            elif status == '' and category == '':
                orders = Order.objects.filter(type_of_order='h', hardwaresoftware__origin=origin)
            elif status == '' and origin == '':
                orders = Order.objects.filter(type_of_order='h', hardwaresoftware__category=category)
            elif category == '' and origin == '':
                orders = Order.objects.filter(type_of_order='h', status=status)
            elif status == '':
                orders = Order.objects.filter(type_of_order='h', hardwaresoftware__origin=origin,
                                              hardwaresoftware__category=category)
            elif category == '':
                orders = Order.objects.filter(type_of_order='h', status=status, hardwaresoftware__origin=origin)
            elif origin == '':
                orders = Order.objects.filter(type_of_order='h', status=status, hardwaresoftware__category=category)
            else:
                orders = Order.objects.filter(type_of_order='h', status=status, hardwaresoftware__category=category,
                                              hardwaresoftware__origin=origin)

            context = {'orders': orders, 'status': status, 'category': category, 'origin': origin}
            return render(request, 'report/list_equipment_supplies_msc.html', context)

        elif request.POST['order_type'] == "s":
            status = request.POST.get('status')
            origin = request.POST.get('origin')
            if status == '' and origin == '':
                orders = Order.objects.filter(type_of_order='s')
            elif status == '':
                orders = Order.objects.filter(type_of_order='s', service__origin=origin)
            elif origin == '':
                orders = Order.objects.filter(type_of_order='s', status=status)
            else:
                orders = Order.objects.filter(type_of_order='s', status=status, service__origin=origin)

            context = {'orders': orders}
            return render(request, 'report/list_services.html', context)

        elif request.POST['order_type'] == "e":
            status = request.POST.get('status')
            if status == '':
                orders = Order.objects.filter(type_of_order='e')
            else:
                orders = Order.objects.filter(type_of_order='e', status=status)

            context = {'orders': orders}
            return render(request, 'report/list_events.html', context)

        elif request.POST['order_type'] == "t":
            status = request.POST.get('status')
            if status == '':
                orders = Order.objects.filter(type_of_order='t')
            else:
                orders = Order.objects.filter(type_of_order='t', status=status)

            context = {'orders': orders}
            return render(request, 'report/list_tickets.html', context)

        elif request.POST['order_type'] == "d":
            status = request.POST.get('status')
            if status == '':
                orders = Order.objects.filter(type_of_order='d')
            else:
                orders = Order.objects.filter(type_of_order='d', status=status)

            context = {'orders': orders}
            return render(request, 'report/list_daily_stipend.html', context)

        elif request.POST['order_type'] == "r":
            status = request.POST.get('status')
            if status == '':
                orders = Order.objects.filter(type_of_order='r')
            else:
                orders = Order.objects.filter(type_of_order='r', status=status)

            context = {'orders': orders}
            return render(request, 'report/list_reimbursement.html', context)

    context = {'types': types}
    return render(request, 'report/list_order.html', context)

@login_required
def select_additional_options(request):
    orders = request.GET.get('order_type')
    if orders == 'h':
        status = [{'value': status[0], 'display': status[1].encode('utf-8')} for status in ORDER_STATUS]
        categories = [{'value': category[0], 'display': category[1].encode('utf-8')} for category in CATEGORY]
        origin = [{'value': orig[0], 'display': orig[1].encode('utf-8')} for orig in ORIGIN]
    elif orders == 's':
        status = [{'value': status[0], 'display': status[1].encode('utf-8')} for status in ORDER_STATUS]
        origin = [{'value': orig[0], 'display': orig[1].encode('utf-8')} for orig in ORIGIN]
        categories = []
    else:
        status = [{'value': status[0], 'display': status[1].encode('utf-8')} for status in ORDER_STATUS]
        categories = []
        origin = []
    return HttpResponse(json.dumps([status, categories, origin]), content_type="application/json")<|endoftext|>


<commit_before>from telegram import ParseMode
from sqlalchemy import Column, Integer, Float, ForeignKey, String, create_engine
from sqlalchemy.orm import relationship
import re

import config
from coin import Coin, get_coin_from_input
from user import get_or_create_user
from db_base import Base, Session, engine, session

import logging

Base.metadata.bind = engine

class Wallet(Base):
    __tablename__ = 'wallet'

    id = Column(Integer, primary_key=True)
    user_id = Column(Integer, ForeignKey('user.id'))
    coin_id = Column(Integer, ForeignKey('coin.id'))
    user = relationship("User", back_populates="wallets")
    coin = relationship("Coin", back_populates="wallets")

    amount = Column(Integer, nullable=False)
    send_addr = Column(String, nullable=True)
    recieve_addr = Column(String, nullable=True)
    stored_on = Column(String(50), nullable=True)

    def __init__(self, user, coin, amount, send = None, recieve = None, store = None):
        self.user = user
        self.coin = coin
        self.amount = amount
        self.send_addr = send
        self.recieve_addr = recieve
        self.stored_on = store

def wallet(bot, update):
    user = get_or_create_user(update.message.from_user)
    wallets = session.query(Wallet).filter((Wallet.user == user) &\
                                     (Wallet.amount > 0)).all()
    if len(wallets) == 0:
        message = "You have no coins in your portfolio"
    else:
        message = u"{}'s Portfolio".format(user.first_name)
        total_value = 0
        total_change = 0
        for w in wallets:
            amount_val = w.coin.price_usd*w.amount
            dollar_change = amount_val*(w.coin.change_24h/100)

            total_value += amount_val
            total_change += dollar_change

            message += u"\n{} {}: ${:,} ({:+}% \u2192 ${:+,})".format(w.amount, w.coin.symbol, round(amount_val,2), w.coin.change_24h, round(dollar_change,2))
        message += u"\n\nUSD Change over 24h: ${:,}".format(round(total_change,2))
        message += u"\nTotal Value: ${:,}".format(round(total_value,2))
    bot.send_message(chat_id=update.message.chat_id,
                     text=message,
                     parse_mode=ParseMode.HTML)

def wallet_message_handler(bot, update):
    message_list = update.message.text.split()
    user = get_or_create_user(update.message.from_user)
    if message_list[0].lower() not in ['add', 'bought', 'buy', 'sold', 'sell', 'remove', 'trade', 'traded', 'exchanged', 'exchange']:
        return

    # This will take the message, convert it into an action and return the
    # response from the action
    message = get_message(message_list, user)
    if message is None:
        return
    update.message.reply_text(message)

def get_message(input, user):
    # We do this so that we can not accidentally add
    if input[0].lower() in ['add', 'bought', 'buy']:
        message = add_coin(user, input)
    elif input[0].lower() in ['sold', 'sell', 'remove']:
        message = remove_coin(user, input)
    elif input[0].lower() in ['trade', 'traded', 'exchange', 'exchanged']:
        message = trade_coin(user, input)
    else:
        return None
    return message

def add_coin(user, input):
    num, coin_string = get_num_coins(input)
    coin = get_coin_from_input(coin_string)

    wallet = session.query(Wallet).filter((Wallet.user == user) &\
                                          (Wallet.coin == coin)).first()

    # if the user has this coin simply add to it, otherwise set it up
    if wallet:
        wallet.amount += num
        session.commit()
    else:
        wallet = Wallet(user, coin, num)
        session.add(wallet)
        session.commit()
    return "{} {} added".format(round(num,3), coin.symbol)

def remove_coin(user, input):
    num, coin_string = get_num_coins(input)
    coin = get_coin_from_input(coin_string)

    wallet = session.query(Wallet).filter((Wallet.user == user) &\
                                          (Wallet.coin == coin)).first()
    if wallet:
        if num == "all":
            return_string = "You have sold all of your {}".format(coin.symbol)
            wallet.amount = 0.0
            session.commit()
            return return_string
        else:
            wallet.amount -= num
            session.commit()
            return "You have sold {} of your {}. {} {} remaining".format(round(num,3), coin.symbol, round(wallet.amount,3), coin.symbol)
    else:
        # dont remove someting thats not there
        return "You have no {} to remove you big dumb baby".format(coin.symbol)

def trade_coin(user, input):
    message = "The trading format is 'trade 10 eth for 1000 gnt'"
    if len(input) < 6:
        return message
    else:
        from_num, from_str = get_num_coins(input[:3])
        to_num, to_str = get_num_coins(input[3:])

        from_coin = get_coin_from_input(from_str)
        to_coin = get_coin_from_input(to_str)

        w_from_coin = session.query(Wallet).filter((Wallet.user == user) &\
                                                   (Wallet.coin == from_coin)).first()
        w_to_coin = session.query(Wallet).filter((Wallet.user == user) &\
                                                 (Wallet.coin == to_coin)).first()

        if w_from_coin:
            w_from_coin.amount -= from_num
            if w_to_coin:
                w_to_coin.amount += from_num
            else:
                new_wallet = Wallet(user, to_coin, to_num)
                session.add(new_wallet)
            session.commit()
            message = "You traded {} {} for {} {}".format(from_num, from_coin.symbol, to_num, to_coin.symbol)
        else:
            pass
    return message

def get_num_coins(input):
    if len(input) == 3:
        if input[1] == 'all':
            num = input[1]
        else:
            num = float(input[1])
        coin = input[2].upper()
    else:
        num_coins = re.findall('\d+|\D+', input[1])
        num = float(num_coins[0])
        coin = num_coins[1].upper()

    return num, coin

<commit_msg>small addition bug<commit_after>from telegram import ParseMode
from sqlalchemy import Column, Integer, Float, ForeignKey, String, create_engine
from sqlalchemy.orm import relationship
import re

import config
from coin import Coin, get_coin_from_input
from user import get_or_create_user
from db_base import Base, Session, engine, session

import logging

Base.metadata.bind = engine

class Wallet(Base):
    __tablename__ = 'wallet'

    id = Column(Integer, primary_key=True)
    user_id = Column(Integer, ForeignKey('user.id'))
    coin_id = Column(Integer, ForeignKey('coin.id'))
    user = relationship("User", back_populates="wallets")
    coin = relationship("Coin", back_populates="wallets")

    amount = Column(Integer, nullable=False)
    send_addr = Column(String, nullable=True)
    recieve_addr = Column(String, nullable=True)
    stored_on = Column(String(50), nullable=True)

    def __init__(self, user, coin, amount, send = None, recieve = None, store = None):
        self.user = user
        self.coin = coin
        self.amount = amount
        self.send_addr = send
        self.recieve_addr = recieve
        self.stored_on = store

def wallet(bot, update):
    user = get_or_create_user(update.message.from_user)
    wallets = session.query(Wallet).filter((Wallet.user == user) &\
                                     (Wallet.amount > 0)).all()
    if len(wallets) == 0:
        message = "You have no coins in your portfolio"
    else:
        message = u"{}'s Portfolio".format(user.first_name)
        total_value = 0
        total_change = 0
        for w in wallets:
            amount_val = w.coin.price_usd*w.amount
            dollar_change = amount_val*(w.coin.change_24h/100)

            total_value += amount_val
            total_change += dollar_change

            message += u"\n{} {}: ${:,} ({:+}% \u2192 ${:+,})".format(w.amount, w.coin.symbol, round(amount_val,2), w.coin.change_24h, round(dollar_change,2))
        message += u"\n\nUSD Change over 24h: ${:,}".format(round(total_change,2))
        message += u"\nTotal Value: ${:,}".format(round(total_value,2))
    bot.send_message(chat_id=update.message.chat_id,
                     text=message,
                     parse_mode=ParseMode.HTML)

def wallet_message_handler(bot, update):
    message_list = update.message.text.split()
    user = get_or_create_user(update.message.from_user)
    if message_list[0].lower() not in ['add', 'bought', 'buy', 'sold', 'sell', 'remove', 'trade', 'traded', 'exchanged', 'exchange']:
        return

    # This will take the message, convert it into an action and return the
    # response from the action
    message = get_message(message_list, user)
    if message is None:
        return
    update.message.reply_text(message)

def get_message(input, user):
    # We do this so that we can not accidentally add
    if input[0].lower() in ['add', 'bought', 'buy']:
        message = add_coin(user, input)
    elif input[0].lower() in ['sold', 'sell', 'remove']:
        message = remove_coin(user, input)
    elif input[0].lower() in ['trade', 'traded', 'exchange', 'exchanged']:
        message = trade_coin(user, input)
    else:
        return None
    return message

def add_coin(user, input):
    num, coin_string = get_num_coins(input)
    coin = get_coin_from_input(coin_string)

    wallet = session.query(Wallet).filter((Wallet.user == user) &\
                                          (Wallet.coin == coin)).first()

    # if the user has this coin simply add to it, otherwise set it up
    if wallet:
        wallet.amount += num
        session.commit()
    else:
        wallet = Wallet(user, coin, num)
        session.add(wallet)
        session.commit()
    return "{} {} added".format(round(num,3), coin.symbol)

def remove_coin(user, input):
    num, coin_string = get_num_coins(input)
    coin = get_coin_from_input(coin_string)

    wallet = session.query(Wallet).filter((Wallet.user == user) &\
                                          (Wallet.coin == coin)).first()
    if wallet:
        if num == "all":
            return_string = "You have sold all of your {}".format(coin.symbol)
            wallet.amount = 0.0
            session.commit()
            return return_string
        else:
            wallet.amount -= num
            session.commit()
            return "You have sold {} of your {}. {} {} remaining".format(round(num,3), coin.symbol, round(wallet.amount,3), coin.symbol)
    else:
        # dont remove someting thats not there
        return "You have no {} to remove you big dumb baby".format(coin.symbol)

def trade_coin(user, input):
    message = "The trading format is 'trade 10 eth for 1000 gnt'"
    if len(input) < 6:
        return message
    else:
        from_num, from_str = get_num_coins(input[:3])
        to_num, to_str = get_num_coins(input[3:])

        from_coin = get_coin_from_input(from_str)
        to_coin = get_coin_from_input(to_str)

        w_from_coin = session.query(Wallet).filter((Wallet.user == user) &\
                                                   (Wallet.coin == from_coin)).first()
        w_to_coin = session.query(Wallet).filter((Wallet.user == user) &\
                                                 (Wallet.coin == to_coin)).first()

        if w_from_coin:
            w_from_coin.amount -= from_num
            if w_to_coin:
                w_to_coin.amount += to_num
            else:
                new_wallet = Wallet(user, to_coin, to_num)
                session.add(new_wallet)
            session.commit()
            message = "You traded {} {} for {} {}".format(from_num, from_coin.symbol, to_num, to_coin.symbol)
        else:
            pass
    return message

def get_num_coins(input):
    if len(input) == 3:
        if input[1] == 'all':
            num = input[1]
        else:
            num = float(input[1])
        coin = input[2].upper()
    else:
        num_coins = re.findall('\d+|\D+', input[1])
        num = float(num_coins[0])
        coin = num_coins[1].upper()

    return num, coin

<|endoftext|>