<commit_before>#!/usr/bin/python
import mechanize
import urllib
import sys
import logging
import traceback
import simplejson
import re
import time
import subprocess
import optparse
import os
import datetime
import textwrap
from urllib2 import HTTPError

sys.path.append("/mit/tabbott/Public/python-zephyr/")
sys.path.append("/mit/tabbott/Public/python-zephyr/build/lib.linux-x86_64-2.6/")
import zephyr

zephyr.init()

parser = optparse.OptionParser()
parser.add_option('--forward-class-messages',
                  dest='forward_class_messages',
                  default=False,
                  action='store_true')
parser.add_option('--resend-log',
                  dest='resend_log',
                  default=False,
                  action='store_true')
parser.add_option('--enable-log',
                  dest='enable_log',
                  default=False,
                  action='store_true')
parser.add_option('--no-forward-personals',
                  dest='forward_personals',
                  default=True,
                  action='store_false')
parser.add_option('--forward-from-humbug',
                  dest='forward_to_humbug',
                  default=True,
                  action='store_false')
(options, args) = parser.parse_args()

browser = None
csrf_token = None

def browser_login():
    logger = logging.getLogger("mechanize")
    logger.addHandler(logging.StreamHandler(sys.stdout))
    logger.setLevel(logging.INFO)

    global browser
    browser = mechanize.Browser()
    browser.set_handle_robots(False)
    ## debugging code to consider
    # browser.set_debug_http(True)
    # browser.set_debug_responses(True)
    # browser.set_debug_redirects(True)
    # browser.set_handle_refresh(False)

    browser.add_password("https://app.humbughq.com/", "tabbott", "xxxxxxxxxxxxxxxxx", "wiki")
    browser.open("https://app.humbughq.com/")
    browser.follow_link(text_regex="\s*Log in\s*")
    browser.select_form(nr=0)
    browser["username"] = os.environ["USER"] + "@mit.edu"
    browser["password"] = os.environ["USER"]

    global csrf_token
    csrf_token = browser["csrfmiddlewaretoken"]

    browser.submit()

def send_humbug(zeph):
    zeph["sender"] = zeph["sender"].lower().replace("athena.mit.edu", "mit.edu")
    if "recipient" in zeph:
        zeph["recipient"] = zeph["recipient"].lower().replace("athena.mit.edu", "mit.edu")
    zeph['fullname']  = username_to_fullname(zeph['sender'])
    zeph['shortname'] = zeph['sender'].split('@')[0]
    if "instance" in zeph:
        zeph["instance"] = zeph["instance"][:30]
    browser.addheaders.append(('X-CSRFToken', csrf_token))

    humbug_data = []
    for key in zeph.keys():
        if isinstance(zeph[key], unicode):
            val = zeph[key].encode("utf-8")
        elif isinstance(zeph[key], str):
            val = zeph[key].decode("utf-8")
        humbug_data.append((key, val))

    try:
        browser.open("https://app.humbughq.com/forge_zephyr/", urllib.urlencode(humbug_data))
    except HTTPError, e:
        if e.code == 401:
            # Digest auth failed; server was probably restarted; login in again
            while True:
                try:
                    browser_login()
                except HTTPError, e:
                    print "Failed logging in; trying again in 10 seconds."
                    time.sleep(10)
                    continue
                break

            print "Auth failure; trying again after logging in a second time!"
            browser.open("https://app.humbughq.com/forge_zephyr/", urllib.urlencode(humbug_data))
        else:
            raise

def fetch_fullname(username):
    try:
        match_user = re.match(r'([a-zA-Z0-9_]+)@mit\.edu', username)
        if match_user:
            proc = subprocess.Popen(['hesinfo', match_user.group(1), 'passwd'], stdout=subprocess.PIPE)
            out, _err_unused = proc.communicate()
            if proc.returncode == 0:
                return out.split(':')[4].split(',')[0]
    except:
        print >>sys.stderr, 'Error getting fullname for', username
        traceback.print_exc()

    return username.title().replace('@', ' at ').replace('.', ' dot ')

fullnames = {}
def username_to_fullname(username):
    if username not in fullnames:
        fullnames[username] = fetch_fullname(username)
    return fullnames[username]

def process_loop(log):
    import mit_subs_list
    while True:
        try:
            notice = zephyr.receive(block=True)
            zsig, body = notice.message.split("\x00", 1)
            is_personal = False
            is_huddle = False
            if isinstance(zsig, str):
                # Check for width unicode character u'\u200B'.encode("utf-8")
                if u'\u200B'.encode("utf-8") in zsig:
                    print "Skipping message from Humbug!"
                    continue

            sender = notice.sender.lower().replace("athena.mit.edu", "mit.edu")
            recipient = notice.recipient.lower().replace("athena.mit.edu", "mit.edu")

            if (notice.cls.lower() == "message" and
                notice.instance.lower() == "personal"):
                is_personal = True
                if body.startswith("CC:"):
                    is_huddle = True
                    # Map "CC: sipbtest espuser" => "starnine@mit.edu,espuser@mit.edu"
                    huddle_recipients_list = [x + "@mit.edu" for x in
                                              body.split("\n")[0][4:].split()]
                    if sender not in huddle_recipients_list:
                        huddle_recipients_list.append(sender)
                    huddle_recipients = ",".join(huddle_recipients_list)

            if notice.opcode != "":
                # skip PING messages
                continue

            # Drop messages not to the listed subscriptions
            if (notice.cls.lower() not in mit_subs_list.all_subs) and not \
                    (is_personal and options.forward_personals):
                print "Skipping ...", notice.cls, notice.instance, is_personal
                continue

            if is_huddle:
                zeph = { 'type'      : 'personal',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'recipient' : huddle_recipients,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body.split("\n", 1)[1] }
            elif is_personal:
                zeph = { 'type'      : 'personal',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'recipient' : recipient,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body }
            else:
                zeph = { 'type'      : 'class',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'class'     : notice.cls.lower(),
                         'instance'  : notice.instance,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body }

            log.write(simplejson.dumps(zeph) + '\n')
            log.flush()

            print "received a message on %s/%s from %s..." % \
                (notice.cls, notice.instance, notice.sender)
            send_humbug(zeph)
        except:
            print >>sys.stderr, 'Error relaying zephyr'
            traceback.print_exc()
            time.sleep(2)

def zephyr_to_humbug(options):
    browser_login()

    import mit_subs_list
    subs = zephyr.Subscriptions()
    if options.forward_class_messages:
        for sub in mit_subs_list.all_subs:
            subs.add((sub, '*', '*'))
    if options.forward_personals:
        subs.add(("message", "personal", "*"))

    if options.resend_log:
        with open('zephyrs', 'r') as log:
            for ln in log:
                try:
                    zeph = simplejson.loads(ln)
                    print "sending saved message to %s from %s..." % \
                        (zeph.get('class', zeph.get('recipient')), zeph['sender'])
                    send_humbug(zeph)
                except:
                    print >>sys.stderr, 'Could not send saved zephyr'
                    traceback.print_exc()
                    time.sleep(2)

    print "Starting receive loop"

    if options.enable_log:
        log_file = "zephyrs"
    else:
        log_file = "/dev/null"

    with open(log_file, 'a') as log:
        process_loop(log)

def get_new_zephyrs():
        browser.addheaders.append(('X-CSRFToken', csrf_token))
        submit_hash = {"mit_sync_bot": 'yes'}
        submit_data = urllib.urlencode([(k, v.encode('utf-8')) for k,v in submit_hash.items()])
        res = browser.open("https://app.humbughq.com/api/get_updates", submit_data)
        return simplejson.loads(res.read())['zephyrs']

def send_zephyr(message):
    zsig = u"%s\u200B" % (username_to_fullname(message["sender_email"]))
    if ' dot ' in zsig:
        print "ERROR!  Couldn't compute zsig for %s!" % (message["sender_email"])
        return
    wrapped_content = "\n".join(textwrap.wrap(message["content"]))
    print "Sending message from %s humbug=>zephyr at %s" % (message["sender_email"], datetime.datetime.now())
    if message['type'] == "class":
        zeph = zephyr.ZNotice(sender=message["sender_email"].replace("mit.edu", "ATHENA.MIT.EDU"),
                              auth=True, cls=message["display_recipient"],
                              instance=message["instance"])
        body = "%s\0%s" % (zsig, wrapped_content)
        zeph.setmessage(body)
        zeph.send()
    elif message['type'] == "personal":
        zeph = zephyr.ZNotice(sender=message["sender_email"].replace("mit.edu", "ATHENA.MIT.EDU"),
                              auth=True, recipient=message["display_recipient"].replace("mit.edu", "ATHENA.MIT.EDU"),
                              cls="message", instance="personal")
        body = "%s\0%s" % (zsig, wrapped_content)
        zeph.setmessage(body)
        zeph.send()
    elif message['type'] == "huddle":
        cc_list = ["CC:"]
        cc_list.extend([user["email"].replace("@mit.edu", "")
                        for user in message["display_recipient"]])
        body = "%s\0%s\n%s" % (zsig, " ".join(cc_list), wrapped_content)
        for r in message["display_recipient"]:
            zeph = zephyr.ZNotice(sender=message["sender_email"].replace("mit.edu", "ATHENA.MIT.EDU"),
                                  auth=True, recipient=r["email"].replace("mit.edu", "ATHENA.MIT.EDU"),
                                  cls="message", instance="personal")
            zeph.setmessage(body)
            zeph.send()

def humbug_to_zephyr(options):
    # Sync messages from zephyr to humbug
    browser_login()
    print "Starting syncing messages."
    while True:
        try:
            zephyrs = get_new_zephyrs()
        except HTTPError, e:
            # 502/503 typically means the server was restarted; sleep
            # a bit, then try again
            time.sleep(1)
            if e.code == 401:
                # 401 means digest auth failed -- we need to login again
                while True:
                    try:
                        browser_login()
                    except HTTPError, e:
                        print "Failed logging in; trying again in 10 seconds."
                        time.sleep(10)
                        continue
                    break
            continue
        for zephyr in zephyrs:
            if zephyr["sender_email"] == os.environ["USER"] + "@mit.edu":
                if float(zephyr["timestamp"]) < float(datetime.datetime.now().strftime("%s")) - 5:
                    print "Alert!  Out of order message!", zephyr["timestamp"], datetime.datetime.now().strftime("%s")
                    continue
                send_zephyr(zephyr)

if options.forward_to_humbug:
    zephyr_to_humbug(options)
else:
    humbug_to_zephyr(options)
<commit_msg>zephyr_mirror: Add print statement for message download retries.<commit_after>#!/usr/bin/python
import mechanize
import urllib
import sys
import logging
import traceback
import simplejson
import re
import time
import subprocess
import optparse
import os
import datetime
import textwrap
from urllib2 import HTTPError

sys.path.append("/mit/tabbott/Public/python-zephyr/")
sys.path.append("/mit/tabbott/Public/python-zephyr/build/lib.linux-x86_64-2.6/")
import zephyr

zephyr.init()

parser = optparse.OptionParser()
parser.add_option('--forward-class-messages',
                  dest='forward_class_messages',
                  default=False,
                  action='store_true')
parser.add_option('--resend-log',
                  dest='resend_log',
                  default=False,
                  action='store_true')
parser.add_option('--enable-log',
                  dest='enable_log',
                  default=False,
                  action='store_true')
parser.add_option('--no-forward-personals',
                  dest='forward_personals',
                  default=True,
                  action='store_false')
parser.add_option('--forward-from-humbug',
                  dest='forward_to_humbug',
                  default=True,
                  action='store_false')
(options, args) = parser.parse_args()

browser = None
csrf_token = None

def browser_login():
    logger = logging.getLogger("mechanize")
    logger.addHandler(logging.StreamHandler(sys.stdout))
    logger.setLevel(logging.INFO)

    global browser
    browser = mechanize.Browser()
    browser.set_handle_robots(False)
    ## debugging code to consider
    # browser.set_debug_http(True)
    # browser.set_debug_responses(True)
    # browser.set_debug_redirects(True)
    # browser.set_handle_refresh(False)

    browser.add_password("https://app.humbughq.com/", "tabbott", "xxxxxxxxxxxxxxxxx", "wiki")
    browser.open("https://app.humbughq.com/")
    browser.follow_link(text_regex="\s*Log in\s*")
    browser.select_form(nr=0)
    browser["username"] = os.environ["USER"] + "@mit.edu"
    browser["password"] = os.environ["USER"]

    global csrf_token
    csrf_token = browser["csrfmiddlewaretoken"]

    browser.submit()

def send_humbug(zeph):
    zeph["sender"] = zeph["sender"].lower().replace("athena.mit.edu", "mit.edu")
    if "recipient" in zeph:
        zeph["recipient"] = zeph["recipient"].lower().replace("athena.mit.edu", "mit.edu")
    zeph['fullname']  = username_to_fullname(zeph['sender'])
    zeph['shortname'] = zeph['sender'].split('@')[0]
    if "instance" in zeph:
        zeph["instance"] = zeph["instance"][:30]
    browser.addheaders.append(('X-CSRFToken', csrf_token))

    humbug_data = []
    for key in zeph.keys():
        if isinstance(zeph[key], unicode):
            val = zeph[key].encode("utf-8")
        elif isinstance(zeph[key], str):
            val = zeph[key].decode("utf-8")
        humbug_data.append((key, val))

    try:
        browser.open("https://app.humbughq.com/forge_zephyr/", urllib.urlencode(humbug_data))
    except HTTPError, e:
        if e.code == 401:
            # Digest auth failed; server was probably restarted; login in again
            while True:
                try:
                    browser_login()
                except HTTPError, e:
                    print "Failed logging in; trying again in 10 seconds."
                    time.sleep(10)
                    continue
                break

            print "Auth failure; trying again after logging in a second time!"
            browser.open("https://app.humbughq.com/forge_zephyr/", urllib.urlencode(humbug_data))
        else:
            raise

def fetch_fullname(username):
    try:
        match_user = re.match(r'([a-zA-Z0-9_]+)@mit\.edu', username)
        if match_user:
            proc = subprocess.Popen(['hesinfo', match_user.group(1), 'passwd'], stdout=subprocess.PIPE)
            out, _err_unused = proc.communicate()
            if proc.returncode == 0:
                return out.split(':')[4].split(',')[0]
    except:
        print >>sys.stderr, 'Error getting fullname for', username
        traceback.print_exc()

    return username.title().replace('@', ' at ').replace('.', ' dot ')

fullnames = {}
def username_to_fullname(username):
    if username not in fullnames:
        fullnames[username] = fetch_fullname(username)
    return fullnames[username]

def process_loop(log):
    import mit_subs_list
    while True:
        try:
            notice = zephyr.receive(block=True)
            zsig, body = notice.message.split("\x00", 1)
            is_personal = False
            is_huddle = False
            if isinstance(zsig, str):
                # Check for width unicode character u'\u200B'.encode("utf-8")
                if u'\u200B'.encode("utf-8") in zsig:
                    print "Skipping message from Humbug!"
                    continue

            sender = notice.sender.lower().replace("athena.mit.edu", "mit.edu")
            recipient = notice.recipient.lower().replace("athena.mit.edu", "mit.edu")

            if (notice.cls.lower() == "message" and
                notice.instance.lower() == "personal"):
                is_personal = True
                if body.startswith("CC:"):
                    is_huddle = True
                    # Map "CC: sipbtest espuser" => "starnine@mit.edu,espuser@mit.edu"
                    huddle_recipients_list = [x + "@mit.edu" for x in
                                              body.split("\n")[0][4:].split()]
                    if sender not in huddle_recipients_list:
                        huddle_recipients_list.append(sender)
                    huddle_recipients = ",".join(huddle_recipients_list)

            if notice.opcode != "":
                # skip PING messages
                continue

            # Drop messages not to the listed subscriptions
            if (notice.cls.lower() not in mit_subs_list.all_subs) and not \
                    (is_personal and options.forward_personals):
                print "Skipping ...", notice.cls, notice.instance, is_personal
                continue

            if is_huddle:
                zeph = { 'type'      : 'personal',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'recipient' : huddle_recipients,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body.split("\n", 1)[1] }
            elif is_personal:
                zeph = { 'type'      : 'personal',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'recipient' : recipient,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body }
            else:
                zeph = { 'type'      : 'class',
                         'time'      : str(notice.time),
                         'sender'    : sender,
                         'class'     : notice.cls.lower(),
                         'instance'  : notice.instance,
                         'zsig'      : zsig,  # logged here but not used by app
                         'new_zephyr': body }

            log.write(simplejson.dumps(zeph) + '\n')
            log.flush()

            print "received a message on %s/%s from %s..." % \
                (notice.cls, notice.instance, notice.sender)
            send_humbug(zeph)
        except:
            print >>sys.stderr, 'Error relaying zephyr'
            traceback.print_exc()
            time.sleep(2)

def zephyr_to_humbug(options):
    browser_login()

    import mit_subs_list
    subs = zephyr.Subscriptions()
    if options.forward_class_messages:
        for sub in mit_subs_list.all_subs:
            subs.add((sub, '*', '*'))
    if options.forward_personals:
        subs.add(("message", "personal", "*"))

    if options.resend_log:
        with open('zephyrs', 'r') as log:
            for ln in log:
                try:
                    zeph = simplejson.loads(ln)
                    print "sending saved message to %s from %s..." % \
                        (zeph.get('class', zeph.get('recipient')), zeph['sender'])
                    send_humbug(zeph)
                except:
                    print >>sys.stderr, 'Could not send saved zephyr'
                    traceback.print_exc()
                    time.sleep(2)

    print "Starting receive loop"

    if options.enable_log:
        log_file = "zephyrs"
    else:
        log_file = "/dev/null"

    with open(log_file, 'a') as log:
        process_loop(log)

def get_new_zephyrs():
        browser.addheaders.append(('X-CSRFToken', csrf_token))
        submit_hash = {"mit_sync_bot": 'yes'}
        submit_data = urllib.urlencode([(k, v.encode('utf-8')) for k,v in submit_hash.items()])
        res = browser.open("https://app.humbughq.com/api/get_updates", submit_data)
        return simplejson.loads(res.read())['zephyrs']

def send_zephyr(message):
    zsig = u"%s\u200B" % (username_to_fullname(message["sender_email"]))
    if ' dot ' in zsig:
        print "ERROR!  Couldn't compute zsig for %s!" % (message["sender_email"])
        return
    wrapped_content = "\n".join(textwrap.wrap(message["content"]))
    print "Sending message from %s humbug=>zephyr at %s" % (message["sender_email"], datetime.datetime.now())
    if message['type'] == "class":
        zeph = zephyr.ZNotice(sender=message["sender_email"].replace("mit.edu", "ATHENA.MIT.EDU"),
                              auth=True, cls=message["display_recipient"],
                              instance=message["instance"])
        body = "%s\0%s" % (zsig, wrapped_content)
        zeph.setmessage(body)
        zeph.send()
    elif message['type'] == "personal":
        zeph = zephyr.ZNotice(sender=message["sender_email"].replace("mit.edu", "ATHENA.MIT.EDU"),
                              auth=True, recipient=message["display_recipient"].replace("mit.edu", "ATHENA.MIT.EDU"),
                              cls="message", instance="personal")
        body = "%s\0%s" % (zsig, wrapped_content)
        zeph.setmessage(body)
        zeph.send()
    elif message['type'] == "huddle":
        cc_list = ["CC:"]
        cc_list.extend([user["email"].replace("@mit.edu", "")
                        for user in message["display_recipient"]])
        body = "%s\0%s\n%s" % (zsig, " ".join(cc_list), wrapped_content)
        for r in message["display_recipient"]:
            zeph = zephyr.ZNotice(sender=message["sender_email"].replace("mit.edu", "ATHENA.MIT.EDU"),
                                  auth=True, recipient=r["email"].replace("mit.edu", "ATHENA.MIT.EDU"),
                                  cls="message", instance="personal")
            zeph.setmessage(body)
            zeph.send()

def humbug_to_zephyr(options):
    # Sync messages from zephyr to humbug
    browser_login()
    print "Starting syncing messages."
    while True:
        try:
            zephyrs = get_new_zephyrs()
        except HTTPError, e:
            # 502/503 typically means the server was restarted; sleep
            # a bit, then try again
            print "Failed getting zephyrs; trying again in 5 seconds."
            time.sleep(5)
            if e.code == 401:
                # 401 means digest auth failed -- we need to login again
                while True:
                    try:
                        browser_login()
                    except HTTPError, e:
                        print "Failed logging in; trying again in 10 seconds."
                        time.sleep(10)
                        continue
                    break
            continue
        for zephyr in zephyrs:
            if zephyr["sender_email"] == os.environ["USER"] + "@mit.edu":
                if float(zephyr["timestamp"]) < float(datetime.datetime.now().strftime("%s")) - 5:
                    print "Alert!  Out of order message!", zephyr["timestamp"], datetime.datetime.now().strftime("%s")
                    continue
                send_zephyr(zephyr)

if options.forward_to_humbug:
    zephyr_to_humbug(options)
else:
    humbug_to_zephyr(options)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2012 Victor Semionov
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#  * Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#  * Neither the name of the copyright holder nor the names of the contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import sys
import os
import glob
import getopt
import time

import signal
import multiprocessing

import meta
import params
import core
import mp
import output
import ext
import svc

usage_help = \
"""Usage: {app_name} {{-h | -v | -e | [-D definition [...]] [-o output_dir] conf_file}}

Options:
 -h             print this help message and exit
 -v             print version information and exit
 -e             list installed extensions
 -D             define or override parameters
 -o output_dir  set output directory path (required for graphs)""".format(app_name=meta.app_name)

help_hint = "Try \"{app_name} -h\" for more information.".format(app_name=meta.app_name)

class InvocationError(Exception):
    pass

def load_extensions():
    extension_path = os.path.normpath(os.path.expanduser("~/.%s/extensions" % meta.app_name.lower()))
    sys.path.append(extension_path)
    extension_pathnames = glob.glob(os.path.join(extension_path, "*.py"))
    extensions = []
    for pathname in extension_pathnames:
        _, name = os.path.split(pathname)
        name, _ = os.path.splitext(name)
        print "loading extension \"%s\"" % name
        extension = __import__(name)
        extensions.append(extension)
    return extensions

def print_help():
    print usage_help

def print_info():
    print meta.app_name, meta.app_version
    print meta.app_copyright
    print
    print meta.app_description
    print
    print meta.app_author_msg
    print meta.app_coauthors_msg
    print
    print meta.app_website_msg

def print_extensions(extensions):
    if not extensions:
        print "no extensions installed"
    else:
        print "installed extensions (name: description):"
        for extension in extensions:
            name, doc = extension.__name__, extension.__doc__
            print "%s: %s" % (name, doc)

def run(conf_path, output_path, definitions):
    start_time = time.clock()

    print "configuring"
    if conf_path is not None:
        if params.verbose:
            print "reading configuration from:", conf_path
        execfile(conf_path, params.__dict__)
    if definitions is not None:
        for definition in definitions:
            exec definition in params.__dict__

    print "validating"
    core.validate()

    output.output_dir = output_path
    if params.graphs:
        if not output.output_dir:
            raise InvocationError("no output directory")
        if params.verbose:
            print "graphs will be written to:", output_path
    else:
        if params.verbose:
            print "graphs will not be written"

    dirname = "."

    int_types = params.integrator_classes
    amp_types = params.amplifier_classes

    if params.lower_process_priority:
        svc.lower_process_priority()

    with mp.TaskPool(params.num_tasks, params.extended_mode, params.__dict__) as task_pool:
        if params.verbose:
            print "number of parallel tasks:", task_pool.num_tasks

        print "executing"

        if not params.initial_inversion:
            ref_inversion, ref_inversion_rel_error = core.compute_inversion(dirname)
        else:
            ref_inversion, ref_inversion_rel_error = params.initial_inversion, 0.0

        numerics = None
        if params.amplification:
            numerics, rel_errors = core.setup_methods(dirname, (int_types, amp_types), ref_inversion, ret_rel_errors=True)
            num_types, counts = numerics
            energy_rel_error = core.compute_energy_rel_error(ref_inversion, ref_inversion_rel_error, rel_errors)
            max_output_fluence, output_photon_counts, output_energy, rel_gain_reduction = core.amplify_train(dirname, num_types, counts, ref_inversion)
            core.report_output_characteristics(ref_inversion, max_output_fluence, output_photon_counts, output_energy, rel_gain_reduction, ref_inversion_rel_error, energy_rel_error)

        if params.extended_mode:
            ext.extended_mode(task_pool, dirname, ref_inversion, (int_types, amp_types), numerics)

    print output.div_line
    print "done"

    end_time = time.clock()
    elapsed_time = end_time - start_time
    if params.verbose:
        print "finished in %.3f s" % elapsed_time

def process(extensions):
    try:
        conf_path = None
        output_path = None
        definitions = []

        help_flag = False
        version_flag = False
        extensions_flag = False

        opts, args = getopt.getopt(sys.argv[1:], "hveD:o:")
        for opt, arg in opts:
            if opt == "-h":
                help_flag = True
            elif opt == "-v":
                version_flag = True
            elif opt == "-e":
                extensions_flag = True
            elif opt == "-D":
                definitions.append(arg)
            elif opt == "-o":
                output_path = arg
            else:
                assert False, "unhandled option"
        if args:
            conf_path = args[0]

        if help_flag:
            print_help()
            sys.exit()
        if version_flag:
            print_info()
            sys.exit()
        if extensions_flag:
            print_extensions(extensions)
            sys.exit()

        if len(args) < 1:
            raise InvocationError("no input file")
        elif len(args) > 1:
            raise InvocationError("too many arguments")

        run(conf_path, output_path, definitions)
    except InvocationError as ie:
        print >>sys.stderr, "%s: %s" % (meta.app_name, ie.message)
        print >>sys.stderr, help_hint

def main():
    multiprocessing.freeze_support()

    signal.signal(signal.SIGINT, signal.SIG_DFL)

    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1)

    extensions = load_extensions()
    process(extensions)

if __name__ == "__main__":
    main()
<commit_msg>fix timing<commit_after>#!/usr/bin/env python

# Copyright (C) 2012 Victor Semionov
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#  * Redistributions of source code must retain the above copyright notice, this
#    list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#  * Neither the name of the copyright holder nor the names of the contributors
#    may be used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import sys
import os
import glob
import getopt
import time

import signal
import multiprocessing

import meta
import params
import core
import mp
import output
import ext
import svc

usage_help = \
"""Usage: {app_name} {{-h | -v | -e | [-D definition [...]] [-o output_dir] conf_file}}

Options:
 -h             print this help message and exit
 -v             print version information and exit
 -e             list installed extensions
 -D             define or override parameters
 -o output_dir  set output directory path (required for graphs)""".format(app_name=meta.app_name)

help_hint = "Try \"{app_name} -h\" for more information.".format(app_name=meta.app_name)

class InvocationError(Exception):
    pass

def load_extensions():
    extension_path = os.path.normpath(os.path.expanduser("~/.%s/extensions" % meta.app_name.lower()))
    sys.path.append(extension_path)
    extension_pathnames = glob.glob(os.path.join(extension_path, "*.py"))
    extensions = []
    for pathname in extension_pathnames:
        _, name = os.path.split(pathname)
        name, _ = os.path.splitext(name)
        print "loading extension \"%s\"" % name
        extension = __import__(name)
        extensions.append(extension)
    return extensions

def print_help():
    print usage_help

def print_info():
    print meta.app_name, meta.app_version
    print meta.app_copyright
    print
    print meta.app_description
    print
    print meta.app_author_msg
    print meta.app_coauthors_msg
    print
    print meta.app_website_msg

def print_extensions(extensions):
    if not extensions:
        print "no extensions installed"
    else:
        print "installed extensions (name: description):"
        for extension in extensions:
            name, doc = extension.__name__, extension.__doc__
            print "%s: %s" % (name, doc)

def run(conf_path, output_path, definitions):
    start_time = time.time()

    print "configuring"
    if conf_path is not None:
        if params.verbose:
            print "reading configuration from:", conf_path
        execfile(conf_path, params.__dict__)
    if definitions is not None:
        for definition in definitions:
            exec definition in params.__dict__

    print "validating"
    core.validate()

    output.output_dir = output_path
    if params.graphs:
        if not output.output_dir:
            raise InvocationError("no output directory")
        if params.verbose:
            print "graphs will be written to:", output_path
    else:
        if params.verbose:
            print "graphs will not be written"

    dirname = "."

    int_types = params.integrator_classes
    amp_types = params.amplifier_classes

    if params.lower_process_priority:
        svc.lower_process_priority()

    with mp.TaskPool(params.num_tasks, params.extended_mode, params.__dict__) as task_pool:
        if params.verbose:
            print "number of parallel tasks:", task_pool.num_tasks

        print "executing"

        if not params.initial_inversion:
            ref_inversion, ref_inversion_rel_error = core.compute_inversion(dirname)
        else:
            ref_inversion, ref_inversion_rel_error = params.initial_inversion, 0.0

        numerics = None
        if params.amplification:
            numerics, rel_errors = core.setup_methods(dirname, (int_types, amp_types), ref_inversion, ret_rel_errors=True)
            num_types, counts = numerics
            energy_rel_error = core.compute_energy_rel_error(ref_inversion, ref_inversion_rel_error, rel_errors)
            max_output_fluence, output_photon_counts, output_energy, rel_gain_reduction = core.amplify_train(dirname, num_types, counts, ref_inversion)
            core.report_output_characteristics(ref_inversion, max_output_fluence, output_photon_counts, output_energy, rel_gain_reduction, ref_inversion_rel_error, energy_rel_error)

        if params.extended_mode:
            ext.extended_mode(task_pool, dirname, ref_inversion, (int_types, amp_types), numerics)

    print output.div_line
    print "done"

    end_time = time.time()
    elapsed_time = end_time - start_time
    if params.verbose:
        print "finished in %.2f s" % elapsed_time

def process(extensions):
    try:
        conf_path = None
        output_path = None
        definitions = []

        help_flag = False
        version_flag = False
        extensions_flag = False

        opts, args = getopt.getopt(sys.argv[1:], "hveD:o:")
        for opt, arg in opts:
            if opt == "-h":
                help_flag = True
            elif opt == "-v":
                version_flag = True
            elif opt == "-e":
                extensions_flag = True
            elif opt == "-D":
                definitions.append(arg)
            elif opt == "-o":
                output_path = arg
            else:
                assert False, "unhandled option"
        if args:
            conf_path = args[0]

        if help_flag:
            print_help()
            sys.exit()
        if version_flag:
            print_info()
            sys.exit()
        if extensions_flag:
            print_extensions(extensions)
            sys.exit()

        if len(args) < 1:
            raise InvocationError("no input file")
        elif len(args) > 1:
            raise InvocationError("too many arguments")

        run(conf_path, output_path, definitions)
    except InvocationError as ie:
        print >>sys.stderr, "%s: %s" % (meta.app_name, ie.message)
        print >>sys.stderr, help_hint

def main():
    multiprocessing.freeze_support()

    signal.signal(signal.SIGINT, signal.SIG_DFL)

    sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 1)

    extensions = load_extensions()
    process(extensions)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import curses
import time
import random

class Char:

    def __init__(self, char, color):
        self.char = char
        self.color = color

class Colors:
    pass

class BonusFactory:

    class Bonus:
        char = '\u2580'

        def __init__(self, x, y, color):
            self.x = x
            self.y = y
            self.color = color

    def __init__(self, window):
        self.width = window.width
        self.height = window.height
        self.window = window

    def make(self):
        x = random.randint(1, self.width)
        y = random.randint(1, self.height)
        return BonusFactory.Bonus(x, y, self.window.colors.cl_bwhite)

class Snake:
    char = '\u25CF'
    head = '\u25C9'
    UP = 'w'
    DOWN = 's'
    LEFT = 'a'
    RIGHT = 'd'
    direction = RIGHT
    snake = [(1, 1), (1, 2), (1, 3), (1, 4), (2, 4), (3, 4), (3, 5), (3, 6), (3, 7)]

    def __init__(self, win):
        self.win = win.descriptor
        self.width = win.width
        self.height = win.height

    def update(self):
        last_segment = self.snake[0]
        x, y = last_segment
        dir_func = {
                Snake.RIGHT: lambda x, y : (x + 1, y),
                Snake.LEFT: lambda x, y : (x - 1, y),
                Snake.UP: lambda x, y : (x, y - 1),
                Snake.DOWN: lambda x, y : (x, y + 1),
                }
        fun = dir_func[self.direction]

        nx, ny = fun(x, y)

        if nx < 0:
            nx = self.width - 1
        if nx >= self.width:
            nx = 0
        if ny < 0:
            ny = self.height - 1
        if ny >= self.height:
            ny = 0

        self.snake[0] = (nx, ny)
        snake = self.snake
        a, b = x, y

        for i in range(1, len(self.snake)):
            x, y = snake[i]
            snake[i] = (a, b)
            a, b = x, y

    def go(self, direction):
        if set([self.direction, direction]) == set([Snake.UP, Snake.DOWN]):
            return
        if set([self.direction, direction]) == set([Snake.LEFT, Snake.RIGHT]):
            return
        self.direction = direction

    def draw(self):
        for s in self.snake:
            x, y = s
            self.win.addstr(y, x, self.char, self.snake_color)
        x, y = self.snake[0]
        self.win.addstr(y, x, self.head, self.snake_color)

class Window():
    width = 40
    height = 20
    colors = Colors()

    def __init__(self, descriptor):
        self.descriptor = descriptor
        self.colors.cl_bwhite = curses.color_pair(2) | curses.A_BOLD
        self.colors.cl_dwhite = curses.color_pair(2) | curses.A_DIM
        self.colors.cl_nwhite = curses.color_pair(2)

class App:
    colors = Colors()
    width = 40
    height = 20
    win = None
    snake = None
    bonusFactory = None
    bonuses = list()

    def main(stdscr):
        curses.start_color()
        curses.init_pair(1, curses.COLOR_RED, curses.COLOR_BLACK)
        curses.init_pair(2, curses.COLOR_WHITE, curses.COLOR_BLACK)
        stdscr.clear()
        curses.curs_set(False)

        App.window = Window(stdscr)
        App.win = stdscr
        App.snake = Snake(App.window)
        App.colors.cl_bwhite = curses.color_pair(2) | curses.A_BOLD
        App.colors.cl_dwhite = curses.color_pair(2) | curses.A_DIM
        App.colors.cl_nwhite = curses.color_pair(2)
        App.snake.snake_color = curses.color_pair(1) | curses.A_BOLD

        App.bonusFactory = BonusFactory(App.window)
        App.bonuses.append(App.bonusFactory.make())

        App.field = {(x,y):z for x in range(App.width)
            for y in range(App.height) for z in [Char('.', App.colors.cl_nwhite)]}

        App.draw_field(App.field)
        stdscr.refresh()

        notEnd = True
        App.win.nodelay(True)
        while notEnd:
            App.update()
            App.draw_field(App.field)
            stdscr.refresh()
            time.sleep(0.1)

            key = App.win.getch()
            if key == curses.ERR:
                continue
            if chr(key) in ('w', 'a', 's', 'd'):
                App.snake.go(chr(key))
            notEnd = key != ord('q')

    def __init__(self):
        curses.wrapper(App.main)

    def update():
        App.snake.update()

    def draw_field(field):
        for x in range(App.width):
            for y in range(App.height):
                char = field[(x, y)].char
                color = field[(x, y)].color
                App.win.addstr(y, x, char, color)
        App.snake.draw()
        for bonus in App.bonuses:
            x, y = bonus.x, bonus.y
            App.win.addstr(y, x, bonus.char, bonus.color)

if __name__=='__main__':
     app = App()
<commit_msg>make bonuses in dict<commit_after>import curses
import time
import random

class Char:

    def __init__(self, char, color):
        self.char = char
        self.color = color

class Colors:
    pass

class BonusFactory:

    class Bonus:
        char = '\u2580'

        def __init__(self, x, y, color):
            self.x = x
            self.y = y
            self.color = color

    def __init__(self, window):
        self.width = window.width
        self.height = window.height
        self.window = window

    def make(self):
        x = random.randint(1, self.width)
        y = random.randint(1, self.height)
        return BonusFactory.Bonus(x, y, self.window.colors.cl_bwhite)

class Snake:
    char = '\u25CF'
    head = '\u25C9'
    UP = 'w'
    DOWN = 's'
    LEFT = 'a'
    RIGHT = 'd'
    direction = RIGHT
    snake = [(1, 1), (1, 2), (1, 3), (1, 4), (2, 4), (3, 4), (3, 5), (3, 6), (3, 7)]

    def __init__(self, win):
        self.win = win.descriptor
        self.width = win.width
        self.height = win.height

    def update(self):
        last_segment = self.snake[0]
        x, y = last_segment
        dir_func = {
                Snake.RIGHT: lambda x, y : (x + 1, y),
                Snake.LEFT: lambda x, y : (x - 1, y),
                Snake.UP: lambda x, y : (x, y - 1),
                Snake.DOWN: lambda x, y : (x, y + 1),
                }
        fun = dir_func[self.direction]

        nx, ny = fun(x, y)

        if nx < 0:
            nx = self.width - 1
        if nx >= self.width:
            nx = 0
        if ny < 0:
            ny = self.height - 1
        if ny >= self.height:
            ny = 0

        self.snake[0] = (nx, ny)
        snake = self.snake
        a, b = x, y

        for i in range(1, len(self.snake)):
            x, y = snake[i]
            snake[i] = (a, b)
            a, b = x, y

    def go(self, direction):
        if set([self.direction, direction]) == set([Snake.UP, Snake.DOWN]):
            return
        if set([self.direction, direction]) == set([Snake.LEFT, Snake.RIGHT]):
            return
        self.direction = direction

    def draw(self):
        for s in self.snake:
            x, y = s
            self.win.addstr(y, x, self.char, self.snake_color)
        x, y = self.snake[0]
        self.win.addstr(y, x, self.head, self.snake_color)

class Window():
    width = 40
    height = 20
    colors = Colors()

    def __init__(self, descriptor):
        self.descriptor = descriptor
        self.colors.cl_bwhite = curses.color_pair(2) | curses.A_BOLD
        self.colors.cl_dwhite = curses.color_pair(2) | curses.A_DIM
        self.colors.cl_nwhite = curses.color_pair(2)

class App:
    colors = Colors()
    width = 40
    height = 20
    win = None
    snake = None
    bonusFactory = None
    bonuses = dict()

    def main(stdscr):
        curses.start_color()
        curses.init_pair(1, curses.COLOR_RED, curses.COLOR_BLACK)
        curses.init_pair(2, curses.COLOR_WHITE, curses.COLOR_BLACK)
        stdscr.clear()
        curses.curs_set(False)

        App.window = Window(stdscr)
        App.win = stdscr
        App.snake = Snake(App.window)
        App.colors.cl_bwhite = curses.color_pair(2) | curses.A_BOLD
        App.colors.cl_dwhite = curses.color_pair(2) | curses.A_DIM
        App.colors.cl_nwhite = curses.color_pair(2)
        App.snake.snake_color = curses.color_pair(1) | curses.A_BOLD

        App.bonusFactory = BonusFactory(App.window)
        bonus = App.bonusFactory.make()
        App.bonuses[(bonus.x, bonus.y)] = bonus

        App.field = {(x,y):z for x in range(App.width)
            for y in range(App.height) for z in [Char('.', App.colors.cl_nwhite)]}

        App.draw_field(App.field)
        stdscr.refresh()

        notEnd = True
        App.win.nodelay(True)
        while notEnd:
            App.update()
            App.draw_field(App.field)
            stdscr.refresh()
            time.sleep(0.1)

            key = App.win.getch()
            if key == curses.ERR:
                continue
            if chr(key) in ('w', 'a', 's', 'd'):
                App.snake.go(chr(key))
            notEnd = key != ord('q')

    def __init__(self):
        curses.wrapper(App.main)

    def update():
        App.snake.update()

    def draw_field(field):
        for x in range(App.width):
            for y in range(App.height):
                char = field[(x, y)].char
                color = field[(x, y)].color
                App.win.addstr(y, x, char, color)
        App.snake.draw()
        for bonus in App.bonuses.values():
            x, y = bonus.x, bonus.y
            App.win.addstr(y, x, bonus.char, bonus.color)

    def intersect():
        pass

if __name__=='__main__':
     app = App()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Copyright 2015 Joohyun Lee(ppiazi@gmail.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
import os
import getopt
from FilesInfoReader import *
import FileInfo

__author__ = 'ppiazi'
__version__ = 'v1.1.2'

def printUsage():
    print("FilesInfoReader.py [-f <folder>] [-o <output file>] [-h <crc32|md5|sha1>] [-s] [-a <extension>]")
    print("    Version %s" % __version__)
    print("    Options:")
    print("    -f : set a target folder")
    print("    -o : set a file for result")
    print("    -h : set hash method among crc32, md5, sha1")
    print("    -s, --source-only : read source files only")
    print("    -a : add <extension> into source file extension list")
    print("        Example) -a asm -a pl")

if __name__ == "__main__":

    optlist, args = getopt.getopt(sys.argv[1:], "f:o:h:sa:", ["source-only",])

    p_folder = None
    p_output = None
    p_hash = "crc32"
    p_sourcecode_only = False

    for op, p in optlist:
        if op == "-f":
            p_folder = p
        elif op == "-o":
            p_output = p
        elif op == "-h":
            p_hash = p
        elif op in ("-s", "--source-only"):
            p_sourcecode_only = True
        elif op == "-a":
            source_ext = "." + p
            FileInfo.SOURCE_CODE_EXT.append(source_ext)
            print(FileInfo.SOURCE_CODE_EXT)
        else:
            print("Invalid Argument : %s / %s" % (op, p))

    if p_folder == None or p_output == None:
        printUsage()
        os._exit(1)

    fir = FilesInfoReader(p_hash)
    fir.setRootPath(p_folder)
    fir.iterate(p_sourcecode_only)
    fir.saveAsCsv(p_output)
<commit_msg>버전 1.1.3<commit_after># -*- coding: utf-8 -*-
"""
Copyright 2015 Joohyun Lee(ppiazi@gmail.com)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
import sys
import os
import getopt
from FilesInfoReader import *
import FileInfo

__author__ = 'ppiazi'
__version__ = 'v1.1.3'

def printUsage():
    print("FilesInfoReader.py [-f <folder>] [-o <output file>] [-h <crc32|md5|sha1>] [-s] [-a <extension>]")
    print("    Version %s" % __version__)
    print("    Options:")
    print("    -f : set a target folder")
    print("    -o : set a file for result")
    print("    -h : set hash method among crc32, md5, sha1")
    print("    -s, --source-only : read source files only")
    print("    -a : add <extension> into source file extension list")
    print("        Example) -a asm -a pl")

if __name__ == "__main__":

    optlist, args = getopt.getopt(sys.argv[1:], "f:o:h:sa:", ["source-only",])

    p_folder = None
    p_output = None
    p_hash = "crc32"
    p_sourcecode_only = False

    for op, p in optlist:
        if op == "-f":
            p_folder = p
        elif op == "-o":
            p_output = p
        elif op == "-h":
            p_hash = p
        elif op in ("-s", "--source-only"):
            p_sourcecode_only = True
        elif op == "-a":
            source_ext = "." + p
            FileInfo.SOURCE_CODE_EXT.append(source_ext)
            print(FileInfo.SOURCE_CODE_EXT)
        else:
            print("Invalid Argument : %s / %s" % (op, p))

    if p_folder == None or p_output == None:
        printUsage()
        os._exit(1)

    fir = FilesInfoReader(p_hash)
    fir.setRootPath(p_folder)
    fir.iterate(p_sourcecode_only)
    fir.saveAsCsv(p_output)
<|endoftext|>


<commit_before>
from datetime import timedelta

from django.db import connection, transaction
from django.utils.timezone import now as timezone_now
from zerver.models import (Message, UserMessage, ArchivedMessage, ArchivedUserMessage, Realm,
                           Attachment, ArchivedAttachment)

from typing import Any, List

@transaction.atomic
def move_expired_rows(src_model: Any, raw_query: str, **kwargs: Any) -> None:
    src_db_table = src_model._meta.db_table
    src_fields = ["{}.{}".format(src_db_table, field.column) for field in src_model._meta.fields]
    dst_fields = [field.column for field in src_model._meta.fields]
    sql_args = {
        'src_fields': ','.join(src_fields),
        'dst_fields': ','.join(dst_fields),
        'archive_timestamp': timezone_now()
    }
    sql_args.update(kwargs)
    with connection.cursor() as cursor:
        cursor.execute(
            raw_query.format(**sql_args)
        )

def move_expired_messages_to_archive(realm: Realm) -> None:
    query = """
    INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
    SELECT {src_fields}, '{archive_timestamp}'
    FROM zerver_message
    INNER JOIN zerver_userprofile ON zerver_message.sender_id = zerver_userprofile.id
    LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
    WHERE zerver_userprofile.realm_id = {realm_id}
          AND  zerver_message.pub_date < '{check_date}'
          AND zerver_archivedmessage.id is NULL
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)
    move_expired_rows(Message, query, realm_id=realm.id, check_date=check_date.isoformat())

def move_expired_user_messages_to_archive(realm: Realm) -> None:
    query = """
    INSERT INTO zerver_archivedusermessage ({dst_fields}, archive_timestamp)
    SELECT {src_fields}, '{archive_timestamp}'
    FROM zerver_usermessage
    INNER JOIN zerver_userprofile ON zerver_usermessage.user_profile_id = zerver_userprofile.id
    INNER JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_usermessage.message_id
    LEFT JOIN zerver_archivedusermessage ON zerver_archivedusermessage.id = zerver_usermessage.id
    LEFT JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id
    WHERE zerver_userprofile.realm_id = {realm_id}
        AND  zerver_message.pub_date < '{check_date}'
        AND zerver_archivedusermessage.id is NULL
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)
    move_expired_rows(UserMessage, query, realm_id=realm.id, check_date=check_date.isoformat())

def move_expired_attachments_to_archive(realm: Realm) -> None:
    query = """
       INSERT INTO zerver_archivedattachment ({dst_fields}, archive_timestamp)
       SELECT {src_fields}, '{archive_timestamp}'
       FROM zerver_attachment
       INNER JOIN zerver_attachment_messages
           ON zerver_attachment_messages.attachment_id = zerver_attachment.id
       INNER JOIN zerver_archivedmessage
           ON zerver_archivedmessage.id = zerver_attachment_messages.message_id
       LEFT JOIN zerver_archivedattachment ON zerver_archivedattachment.id = zerver_attachment.id
       WHERE zerver_attachment.realm_id = {realm_id}
            AND zerver_archivedattachment.id IS NULL
       GROUP BY zerver_attachment.id
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)
    move_expired_rows(Attachment, query, realm_id=realm.id, check_date=check_date.isoformat())

def move_expired_attachments_message_rows_to_archive(realm: Realm) -> None:
    query = """
       INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id, archivedmessage_id)
       SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
           zerver_attachment_messages.message_id
       FROM zerver_attachment_messages
       INNER JOIN zerver_attachment
           ON zerver_attachment_messages.attachment_id = zerver_attachment.id
       INNER JOIN zerver_message ON zerver_attachment_messages.message_id = zerver_message.id
       LEFT JOIN zerver_archivedattachment_messages
           ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
       WHERE zerver_attachment.realm_id = {realm_id}
            AND  zerver_message.pub_date < '{check_date}'
            AND  zerver_archivedattachment_messages.id IS NULL
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)
    with connection.cursor() as cursor:
        cursor.execute(query.format(realm_id=realm.id, check_date=check_date.isoformat()))

def delete_expired_messages(realm: Realm) -> None:
    removing_messages = Message.objects.filter(
        usermessage__isnull=True, id__in=ArchivedMessage.objects.all(),
        sender__realm_id=realm.id
    )
    removing_messages.delete()

def delete_expired_user_messages(realm: Realm) -> None:
    removing_user_messages = UserMessage.objects.filter(
        id__in=ArchivedUserMessage.objects.all(),
        user_profile__realm_id=realm.id
    )
    removing_user_messages.delete()

def delete_expired_attachments(realm: Realm) -> None:
    attachments_to_remove = Attachment.objects.filter(
        messages__isnull=True, id__in=ArchivedAttachment.objects.all(),
        realm_id=realm.id
    )
    attachments_to_remove.delete()

def clean_unused_messages() -> None:
    unused_messages = Message.objects.filter(
        usermessage__isnull=True, id__in=ArchivedMessage.objects.all()
    )
    unused_messages.delete()

def move_expired_to_archive() -> None:
    for realm in Realm.objects.filter(message_retention_days__isnull=False).order_by("id"):
        move_expired_messages_to_archive(realm)
        move_expired_user_messages_to_archive(realm)
        move_expired_attachments_to_archive(realm)
        move_expired_attachments_message_rows_to_archive(realm)

def clean_expired() -> None:
    for realm in Realm.objects.filter(message_retention_days__isnull=False).order_by("id"):
        delete_expired_user_messages(realm)
        delete_expired_messages(realm)
        delete_expired_attachments(realm)
    clean_unused_messages()

def archive_messages() -> None:
    move_expired_to_archive()
    clean_expired()

def move_attachment_message_to_archive_by_message(message_ids: List[int]) -> None:
    # Move attachments messages relation table data to archive.
    id_list = ', '.join(str(message_id) for message_id in message_ids)

    query = """
        INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id,
            archivedmessage_id)
        SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
            zerver_attachment_messages.message_id
        FROM zerver_attachment_messages
        LEFT JOIN zerver_archivedattachment_messages
            ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
        WHERE zerver_attachment_messages.message_id in ({message_ids})
            AND  zerver_archivedattachment_messages.id IS NULL
    """
    with connection.cursor() as cursor:
        cursor.execute(query.format(message_ids=id_list))

@transaction.atomic
def move_messages_to_archive(message_ids: List[int]) -> None:
    messages = list(Message.objects.filter(id__in=message_ids).values())
    if not messages:
        raise Message.DoesNotExist

    ArchivedMessage.objects.bulk_create([ArchivedMessage(**message) for message in messages])

    # Move user_messages to the archive.
    user_messages = UserMessage.objects.filter(
        message_id__in=message_ids).exclude(id__in=ArchivedUserMessage.objects.all()).values()
    user_messages_ids = [user_message['id'] for user_message in user_messages]
    ArchivedUserMessage.objects.bulk_create(
        [ArchivedUserMessage(**user_message) for user_message in user_messages]
    )

    # Move attachments to archive
    attachments = Attachment.objects.filter(messages__id__in=message_ids).exclude(
        id__in=ArchivedAttachment.objects.all()).distinct().values()
    ArchivedAttachment.objects.bulk_create(
        [ArchivedAttachment(**attachment) for attachment in attachments]
    )

    move_attachment_message_to_archive_by_message(message_ids)

    # Remove data from main tables
    Message.objects.filter(id__in=message_ids).delete()
    UserMessage.objects.filter(id__in=user_messages_ids, message_id__isnull=True).delete()

    archived_attachments = ArchivedAttachment.objects.filter(messages__id__in=message_ids).distinct()
    Attachment.objects.filter(messages__isnull=True, id__in=archived_attachments).delete()
<commit_msg>retention: Fix name of move_attachment_message_to_archive_by_message.<commit_after>
from datetime import timedelta

from django.db import connection, transaction
from django.utils.timezone import now as timezone_now
from zerver.models import (Message, UserMessage, ArchivedMessage, ArchivedUserMessage, Realm,
                           Attachment, ArchivedAttachment)

from typing import Any, List

@transaction.atomic
def move_expired_rows(src_model: Any, raw_query: str, **kwargs: Any) -> None:
    src_db_table = src_model._meta.db_table
    src_fields = ["{}.{}".format(src_db_table, field.column) for field in src_model._meta.fields]
    dst_fields = [field.column for field in src_model._meta.fields]
    sql_args = {
        'src_fields': ','.join(src_fields),
        'dst_fields': ','.join(dst_fields),
        'archive_timestamp': timezone_now()
    }
    sql_args.update(kwargs)
    with connection.cursor() as cursor:
        cursor.execute(
            raw_query.format(**sql_args)
        )

def move_expired_messages_to_archive(realm: Realm) -> None:
    query = """
    INSERT INTO zerver_archivedmessage ({dst_fields}, archive_timestamp)
    SELECT {src_fields}, '{archive_timestamp}'
    FROM zerver_message
    INNER JOIN zerver_userprofile ON zerver_message.sender_id = zerver_userprofile.id
    LEFT JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_message.id
    WHERE zerver_userprofile.realm_id = {realm_id}
          AND  zerver_message.pub_date < '{check_date}'
          AND zerver_archivedmessage.id is NULL
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)
    move_expired_rows(Message, query, realm_id=realm.id, check_date=check_date.isoformat())

def move_expired_user_messages_to_archive(realm: Realm) -> None:
    query = """
    INSERT INTO zerver_archivedusermessage ({dst_fields}, archive_timestamp)
    SELECT {src_fields}, '{archive_timestamp}'
    FROM zerver_usermessage
    INNER JOIN zerver_userprofile ON zerver_usermessage.user_profile_id = zerver_userprofile.id
    INNER JOIN zerver_archivedmessage ON zerver_archivedmessage.id = zerver_usermessage.message_id
    LEFT JOIN zerver_archivedusermessage ON zerver_archivedusermessage.id = zerver_usermessage.id
    LEFT JOIN zerver_message ON zerver_usermessage.message_id = zerver_message.id
    WHERE zerver_userprofile.realm_id = {realm_id}
        AND  zerver_message.pub_date < '{check_date}'
        AND zerver_archivedusermessage.id is NULL
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)
    move_expired_rows(UserMessage, query, realm_id=realm.id, check_date=check_date.isoformat())

def move_expired_attachments_to_archive(realm: Realm) -> None:
    query = """
       INSERT INTO zerver_archivedattachment ({dst_fields}, archive_timestamp)
       SELECT {src_fields}, '{archive_timestamp}'
       FROM zerver_attachment
       INNER JOIN zerver_attachment_messages
           ON zerver_attachment_messages.attachment_id = zerver_attachment.id
       INNER JOIN zerver_archivedmessage
           ON zerver_archivedmessage.id = zerver_attachment_messages.message_id
       LEFT JOIN zerver_archivedattachment ON zerver_archivedattachment.id = zerver_attachment.id
       WHERE zerver_attachment.realm_id = {realm_id}
            AND zerver_archivedattachment.id IS NULL
       GROUP BY zerver_attachment.id
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)
    move_expired_rows(Attachment, query, realm_id=realm.id, check_date=check_date.isoformat())

def move_expired_attachments_message_rows_to_archive(realm: Realm) -> None:
    query = """
       INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id, archivedmessage_id)
       SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
           zerver_attachment_messages.message_id
       FROM zerver_attachment_messages
       INNER JOIN zerver_attachment
           ON zerver_attachment_messages.attachment_id = zerver_attachment.id
       INNER JOIN zerver_message ON zerver_attachment_messages.message_id = zerver_message.id
       LEFT JOIN zerver_archivedattachment_messages
           ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
       WHERE zerver_attachment.realm_id = {realm_id}
            AND  zerver_message.pub_date < '{check_date}'
            AND  zerver_archivedattachment_messages.id IS NULL
    """
    assert realm.message_retention_days is not None
    check_date = timezone_now() - timedelta(days=realm.message_retention_days)
    with connection.cursor() as cursor:
        cursor.execute(query.format(realm_id=realm.id, check_date=check_date.isoformat()))

def delete_expired_messages(realm: Realm) -> None:
    removing_messages = Message.objects.filter(
        usermessage__isnull=True, id__in=ArchivedMessage.objects.all(),
        sender__realm_id=realm.id
    )
    removing_messages.delete()

def delete_expired_user_messages(realm: Realm) -> None:
    removing_user_messages = UserMessage.objects.filter(
        id__in=ArchivedUserMessage.objects.all(),
        user_profile__realm_id=realm.id
    )
    removing_user_messages.delete()

def delete_expired_attachments(realm: Realm) -> None:
    attachments_to_remove = Attachment.objects.filter(
        messages__isnull=True, id__in=ArchivedAttachment.objects.all(),
        realm_id=realm.id
    )
    attachments_to_remove.delete()

def clean_unused_messages() -> None:
    unused_messages = Message.objects.filter(
        usermessage__isnull=True, id__in=ArchivedMessage.objects.all()
    )
    unused_messages.delete()

def move_expired_to_archive() -> None:
    for realm in Realm.objects.filter(message_retention_days__isnull=False).order_by("id"):
        move_expired_messages_to_archive(realm)
        move_expired_user_messages_to_archive(realm)
        move_expired_attachments_to_archive(realm)
        move_expired_attachments_message_rows_to_archive(realm)

def clean_expired() -> None:
    for realm in Realm.objects.filter(message_retention_days__isnull=False).order_by("id"):
        delete_expired_user_messages(realm)
        delete_expired_messages(realm)
        delete_expired_attachments(realm)
    clean_unused_messages()

def archive_messages() -> None:
    move_expired_to_archive()
    clean_expired()

def move_attachment_messages_to_archive_by_message(message_ids: List[int]) -> None:
    # Move attachments messages relation table data to archive.
    id_list = ', '.join(str(message_id) for message_id in message_ids)

    query = """
        INSERT INTO zerver_archivedattachment_messages (id, archivedattachment_id,
            archivedmessage_id)
        SELECT zerver_attachment_messages.id, zerver_attachment_messages.attachment_id,
            zerver_attachment_messages.message_id
        FROM zerver_attachment_messages
        LEFT JOIN zerver_archivedattachment_messages
            ON zerver_archivedattachment_messages.id = zerver_attachment_messages.id
        WHERE zerver_attachment_messages.message_id in ({message_ids})
            AND  zerver_archivedattachment_messages.id IS NULL
    """
    with connection.cursor() as cursor:
        cursor.execute(query.format(message_ids=id_list))

@transaction.atomic
def move_messages_to_archive(message_ids: List[int]) -> None:
    messages = list(Message.objects.filter(id__in=message_ids).values())
    if not messages:
        raise Message.DoesNotExist

    ArchivedMessage.objects.bulk_create([ArchivedMessage(**message) for message in messages])

    # Move user_messages to the archive.
    user_messages = UserMessage.objects.filter(
        message_id__in=message_ids).exclude(id__in=ArchivedUserMessage.objects.all()).values()
    user_messages_ids = [user_message['id'] for user_message in user_messages]
    ArchivedUserMessage.objects.bulk_create(
        [ArchivedUserMessage(**user_message) for user_message in user_messages]
    )

    # Move attachments to archive
    attachments = Attachment.objects.filter(messages__id__in=message_ids).exclude(
        id__in=ArchivedAttachment.objects.all()).distinct().values()
    ArchivedAttachment.objects.bulk_create(
        [ArchivedAttachment(**attachment) for attachment in attachments]
    )

    move_attachment_messages_to_archive_by_message(message_ids)

    # Remove data from main tables
    Message.objects.filter(id__in=message_ids).delete()
    UserMessage.objects.filter(id__in=user_messages_ids, message_id__isnull=True).delete()

    archived_attachments = ArchivedAttachment.objects.filter(messages__id__in=message_ids).distinct()
    Attachment.objects.filter(messages__isnull=True, id__in=archived_attachments).delete()
<|endoftext|>


<commit_before>import ftplib
import config
import log
import tarfile
from datetime import datetime
import db

def getRemoteTars():
	try:
		ml_db = db.connect(config.get('db_host','medialib_live'),config.get('db_user','medialib_live'),config.get('db_password','medialib_live'),config.get('db_name','medialib_live'))
		log.logger.debug('Succesfully connected to database ' + config.get('db_host','medialib_live') )
	except Exception as e:
		log.logger.critical('Could not connect to database '+ config.get('db_host','medialib_live'))
		log.logger.debug(e)
		exit(1)

	#q = 'SELECT name,remote_dir FROM tar_file WHERE name LIKE "%_HERBARIUMWAG_%" AND backup_created BETWEEN "' + date + ' 00:00:00" AND "' + date + ' 23:59:59"'
	q = 'SELECT name,remote_dir FROM tar_file WHERE name LIKE "%_HERBARIUMWAG_%" AND backup_created < "2013-08-01"'
	try:
		tars = db.query(ml_db,q)
		log.logger.debug('Succesfully queried to database ' + config.get('db_host','medialib_live') )
		return tars
	except Exception as e:
		log.logger.critical('Could not query to database '+ config.get('db_host','medialib_live'))
		log.logger.debug(e)
		exit(1)

def indexRemoteTars(tar_list):

	try:
		tar_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))
		log.logger.debug('Succesfully connected to database on: ' + config.get('db_host'))
	except Exception as e:
		log.logger.critical('Could not connect database to :' + config.get('db_host'))
		log.logger.debug(e)

	for tar in tar_list:
		try:
			if int(tar['name'].split('_')[2][:8]) < 20130924:
				remote_dir = tar['name'].split('_')[2][:8]
			elif tar['remote_dir'] is None:
				remote_dir = ''
			else:
				remote_dir = tar['remote_dir']

			q = 'INSERT INTO ftp_index (tar,remote_dir) VALUES ("'+tar['name']+'","'+str(remote_dir)+'")'
			db.update(tar_db,q)
			log.logger.debug('Succesfully added this record to the database: ' + tar['name'])
		except Exception as e:
			log.logger.critical('Could not add record ' + tar['name'] + ' this is a CRITICAL -> EXITING')
			exit(1)

def getTar(setsize):
	try:
		ftp_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))
		log.logger.debug('Succesfully connected to database on: ' + config.get('db_host'))
	except Exception as e:
		log.logger.critical('Could not connect database to :' + config.get('db_host'))
		log.logger.debug(e)

	q = 'SELECT * FROM ftp_index WHERE downloaded = 0 ORDER BY id DESC limit ' + str(setsize)
	try:
		tars = db.query(ftp_db,q)
		log.logger.debug('Succesfully queried to database ' + config.get('db_host') )
		return tars
	except Exception as e:
		log.logger.critical('Could not query to database '+ config.get('db_host'))
		log.logger.debug(e)
		exit(1)

	#for tar in tars:
	#	print tar

def openFTP():
	ftp = ftplib.FTP(config.get('ftp_host'))
	try:
		ftp.login(config.get('ftp_username'),config.get('ftp_password'))
		ftp.set_pasv(True)
		log.logger.debug('FTP connection to ' + config.get('ftp_host') + ' succesfull')
		return ftp
	except Exception as e:
		log.logger.critical('Fail during setup of ftp connection to ' + config.get('ftp_host') )
		log.logger.debug(e)
		raise Exception

def closeFTP(ftp):
	try:
		ftp.close()
		log.logger.debug('FTP connection close of ' + config.get('ftp_host') + ' succesfull')
	except Exception as e:
		log.logger.critical('Fail during closing of ftp connection to ' + config.get('ftp_host') )
		log.logger.debug(e)
		raise Exception

def checkRemoteFile(ftp,path,tarfile):
	try:
		ftp.cwd('/'+path)
		log.logger.debug('Succesfully changed dir to /' + path)
	except Exception as e:
		log.logger.critical('Fail during directory change of /' + path )
		log.logger.debug(e)
		raise Exception

	try:
		size = ftp.size(tarfile)
		log.logger.debug('Succesfully checked size of ' + tarfile )
	except Exception as e:
		log.logger.critical('Fail checked size of ' + tarfile )
		log.logger.debug(e)
		raise Exception

	try:
		if size is None:
			return False
		else:
			return size
	except Exception as e:
		log.logger.critical('Could not return size of ' + tarfile )
		log.logger.debug(e)
		raise Exception

def downloadTar(ftp,path,tarfile):
	try:
		ftp.cwd('/'+path)
		log.logger.debug('Succesfully changed dir to /' + path)
	except Exception as e:
		log.logger.critical('Fail during directory change of /' + path )
		log.logger.debug(e)
		raise Exception

	try:
		ftp.retrbinary('RETR ' + tarfile,open(tarfile, 'wb').write)
		log.logger.debug('Succesfully downloaded ' + tarfile)
	except Exception as e:
		log.logger.critical('Fail retriefing file ' + tarfile )
		log.logger.debug(e)
		raise Exception

	ftp_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))

	q = 'UPDATE ftp_index SET downloaded = 1, download_date = "' +str(datetime.now()) +  '" WHERE tar = "'+tarfile+'"'

	try:
		db.update(ftp_db,q)
		log.logger.debug('Succesfully updated ftp_index database with downloaded file')
	except Exception as e:
		log.logger.critical('Could not update ftp_index database')
		log.logger.debug(e)

def extractTar(id):
	tar_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))
	q = 'SELECT tar,filename FROM tar_index WHERE id = ' + str(id)
	tar_info = []
	try:
		tar_info = db.query(tar_db,q)[0]
	except Exception as e:
		log.logger.critical('Could not query tar_index table')
		log.logger.debug(e)
	if len(tar_info) == 0:
		log.logger.error('Could not find this id in tar_index. ID:' + str(id))
		raise Exception
	else:
		try:
			tar = tarfile.open(tar_info['tar'])
			log.logger.debug('Succesfully opened ' + tar_info['tar'])
		except Exception as e:
			log.logger.critical('Fail during opening ' + tar_info['tar'] )
			log.logger.debug(e)
			raise Exception
		try:
			tar.extract(tar_info['filename'])
			#tar.extract(14)
			log.logger.debug('Succesfully extracted ' + tar_info['filename'])
			return tar_info['filename']
		except Exception as e:
			log.logger.critical('Could not extact' + tar_info['filename'] + ' from ' + tar_info['tar'])
			log.logger.debug(e)

def indexTar(tarfilename):

	__MIME__ = ['tif','tiff','jpg','jpeg']

	tar_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))

	try:
		tar = tarfile.open(tarfilename)
		log.logger.debug('Succesfully opened ' + tarfilename)
	except Exception as e:
		log.logger.critical('Fail during opening ' + tarfilename )
		log.logger.debug(e)
		raise Exception

	try:
		members = tar.getnames()
		log.logger.debug('Succesfully indexed tar file ' + tarfilename)
	except Exception as e:
		log.logger.critical('Fail during opening ' + tarfilename )
		log.logger.debug(e)
		raise Exception
	totaller = 0
	for i in range(len(members)):
		member = members[i]
		if member.split('/')[-1].split('.')[-1] in __MIME__:
			totaller = totaller + 1
			fullpath = member
			tar_index = i
			filename = member.split('/')[-1]
			name = filename[0:-len(filename.split('.')[-1])-1]
			t = tarfilename.split('.')[0].split('_')[-1]
			tar_date = datetime(int(t[:4]),int(t[4:6]),int(t[6:8]),int(t[8:10]),int(t[10:12]),int(t[12:]))
			index_date = datetime.now()
			if '-2013-' in name:
				proccessed_2013 = 0
			else:
				proccessed_2013 = 1

			q = 'INSERT INTO tar_index (tar,filename,tar_index,name,index_date,tar_date,processed_2013) VALUES (\
			"' + tarfilename + '",\
			"' + fullpath + '",\
			"' + str(tar_index) + '",\
			"' + name + '",\
			"' + str(index_date) + '",\
			"' + str(tar_date) + '",\
			"' + str(proccessed_2013) + '"\
			)'
			try:
				db.update(tar_db,q)
			except Exception as e:
				log.logger.critical('Could not add record ' + fullpath + ' from ' + tarfilename + ' this is a CRITICAL -> EXITING')
				exit(1)
		log.logger.info('Indexed ' + str(totaller) + ' files from tar: ' + tarfilename)
<commit_msg>some try fixing on restore.extractTar<commit_after>import ftplib
import config
import log
import tarfile
from datetime import datetime
import db

def getRemoteTars():
	try:
		ml_db = db.connect(config.get('db_host','medialib_live'),config.get('db_user','medialib_live'),config.get('db_password','medialib_live'),config.get('db_name','medialib_live'))
		log.logger.debug('Succesfully connected to database ' + config.get('db_host','medialib_live') )
	except Exception as e:
		log.logger.critical('Could not connect to database '+ config.get('db_host','medialib_live'))
		log.logger.debug(e)
		exit(1)

	#q = 'SELECT name,remote_dir FROM tar_file WHERE name LIKE "%_HERBARIUMWAG_%" AND backup_created BETWEEN "' + date + ' 00:00:00" AND "' + date + ' 23:59:59"'
	q = 'SELECT name,remote_dir FROM tar_file WHERE name LIKE "%_HERBARIUMWAG_%" AND backup_created < "2013-08-01"'
	try:
		tars = db.query(ml_db,q)
		log.logger.debug('Succesfully queried to database ' + config.get('db_host','medialib_live') )
		return tars
	except Exception as e:
		log.logger.critical('Could not query to database '+ config.get('db_host','medialib_live'))
		log.logger.debug(e)
		exit(1)

def indexRemoteTars(tar_list):

	try:
		tar_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))
		log.logger.debug('Succesfully connected to database on: ' + config.get('db_host'))
	except Exception as e:
		log.logger.critical('Could not connect database to :' + config.get('db_host'))
		log.logger.debug(e)

	for tar in tar_list:
		try:
			if int(tar['name'].split('_')[2][:8]) < 20130924:
				remote_dir = tar['name'].split('_')[2][:8]
			elif tar['remote_dir'] is None:
				remote_dir = ''
			else:
				remote_dir = tar['remote_dir']

			q = 'INSERT INTO ftp_index (tar,remote_dir) VALUES ("'+tar['name']+'","'+str(remote_dir)+'")'
			db.update(tar_db,q)
			log.logger.debug('Succesfully added this record to the database: ' + tar['name'])
		except Exception as e:
			log.logger.critical('Could not add record ' + tar['name'] + ' this is a CRITICAL -> EXITING')
			exit(1)

def getTar(setsize):
	try:
		ftp_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))
		log.logger.debug('Succesfully connected to database on: ' + config.get('db_host'))
	except Exception as e:
		log.logger.critical('Could not connect database to :' + config.get('db_host'))
		log.logger.debug(e)

	q = 'SELECT * FROM ftp_index WHERE downloaded = 0 ORDER BY id DESC limit ' + str(setsize)
	try:
		tars = db.query(ftp_db,q)
		log.logger.debug('Succesfully queried to database ' + config.get('db_host') )
		return tars
	except Exception as e:
		log.logger.critical('Could not query to database '+ config.get('db_host'))
		log.logger.debug(e)
		exit(1)

	#for tar in tars:
	#	print tar

def openFTP():
	ftp = ftplib.FTP(config.get('ftp_host'))
	try:
		ftp.login(config.get('ftp_username'),config.get('ftp_password'))
		ftp.set_pasv(True)
		log.logger.debug('FTP connection to ' + config.get('ftp_host') + ' succesfull')
		return ftp
	except Exception as e:
		log.logger.critical('Fail during setup of ftp connection to ' + config.get('ftp_host') )
		log.logger.debug(e)
		raise Exception

def closeFTP(ftp):
	try:
		ftp.close()
		log.logger.debug('FTP connection close of ' + config.get('ftp_host') + ' succesfull')
	except Exception as e:
		log.logger.critical('Fail during closing of ftp connection to ' + config.get('ftp_host') )
		log.logger.debug(e)
		raise Exception

def checkRemoteFile(ftp,path,tarfile):
	try:
		ftp.cwd('/'+path)
		log.logger.debug('Succesfully changed dir to /' + path)
	except Exception as e:
		log.logger.critical('Fail during directory change of /' + path )
		log.logger.debug(e)
		raise Exception

	try:
		size = ftp.size(tarfile)
		log.logger.debug('Succesfully checked size of ' + tarfile )
	except Exception as e:
		log.logger.critical('Fail checked size of ' + tarfile )
		log.logger.debug(e)
		raise Exception

	try:
		if size is None:
			return False
		else:
			return size
	except Exception as e:
		log.logger.critical('Could not return size of ' + tarfile )
		log.logger.debug(e)
		raise Exception

def downloadTar(ftp,path,tarfile):
	try:
		ftp.cwd('/'+path)
		log.logger.debug('Succesfully changed dir to /' + path)
	except Exception as e:
		log.logger.critical('Fail during directory change of /' + path )
		log.logger.debug(e)
		raise Exception

	try:
		ftp.retrbinary('RETR ' + tarfile,open(tarfile, 'wb').write)
		log.logger.debug('Succesfully downloaded ' + tarfile)
	except Exception as e:
		log.logger.critical('Fail retriefing file ' + tarfile )
		log.logger.debug(e)
		raise Exception

	ftp_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))

	q = 'UPDATE ftp_index SET downloaded = 1, download_date = "' +str(datetime.now()) +  '" WHERE tar = "'+tarfile+'"'

	try:
		db.update(ftp_db,q)
		log.logger.debug('Succesfully updated ftp_index database with downloaded file')
	except Exception as e:
		log.logger.critical('Could not update ftp_index database')
		log.logger.debug(e)

def extractTar(id):
	tar_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))
	q = 'SELECT tar,filename FROM tar_index WHERE id = ' + str(id)
	#tar_info = []
	try:
		tar_info = db.query(tar_db,q)[0]
	except Exception as e:
		log.logger.critical('Could not query tar_index table')
		log.logger.debug(e)
	if len(tar_info) == 0:
		log.logger.error('Could not find this id in tar_index. ID:' + str(id))
		raise Exception
	else:
		try:
			tar = tarfile.open(tar_info['tar'])
			log.logger.debug('Succesfully opened ' + tar_info['tar'])
		except Exception as e:
			log.logger.critical('Fail during opening ' + tar_info['tar'] )
			log.logger.debug(e)
			raise Exception
		try:
			tar.extract(tar_info['filename'])
			#tar.extract(14)
			log.logger.debug('Succesfully extracted ' + tar_info['filename'])
			return tar_info['filename']
		except Exception as e:
			log.logger.critical('Could not extact' + tar_info['filename'] + ' from ' + tar_info['tar'])
			log.logger.debug(e)

def indexTar(tarfilename):

	__MIME__ = ['tif','tiff','jpg','jpeg']

	tar_db = db.connect(config.get('db_host'),config.get('db_user'),config.get('db_password'),config.get('db_name'))

	try:
		tar = tarfile.open(tarfilename)
		log.logger.debug('Succesfully opened ' + tarfilename)
	except Exception as e:
		log.logger.critical('Fail during opening ' + tarfilename )
		log.logger.debug(e)
		raise Exception

	try:
		members = tar.getnames()
		log.logger.debug('Succesfully indexed tar file ' + tarfilename)
	except Exception as e:
		log.logger.critical('Fail during opening ' + tarfilename )
		log.logger.debug(e)
		raise Exception
	totaller = 0
	for i in range(len(members)):
		member = members[i]
		if member.split('/')[-1].split('.')[-1] in __MIME__:
			totaller = totaller + 1
			fullpath = member
			tar_index = i
			filename = member.split('/')[-1]
			name = filename[0:-len(filename.split('.')[-1])-1]
			t = tarfilename.split('.')[0].split('_')[-1]
			tar_date = datetime(int(t[:4]),int(t[4:6]),int(t[6:8]),int(t[8:10]),int(t[10:12]),int(t[12:]))
			index_date = datetime.now()
			if '-2013-' in name:
				proccessed_2013 = 0
			else:
				proccessed_2013 = 1

			q = 'INSERT INTO tar_index (tar,filename,tar_index,name,index_date,tar_date,processed_2013) VALUES (\
			"' + tarfilename + '",\
			"' + fullpath + '",\
			"' + str(tar_index) + '",\
			"' + name + '",\
			"' + str(index_date) + '",\
			"' + str(tar_date) + '",\
			"' + str(proccessed_2013) + '"\
			)'
			try:
				db.update(tar_db,q)
			except Exception as e:
				log.logger.critical('Could not add record ' + fullpath + ' from ' + tarfilename + ' this is a CRITICAL -> EXITING')
				exit(1)
		log.logger.info('Indexed ' + str(totaller) + ' files from tar: ' + tarfilename)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (C) 2015 Intel Corporation
#
# Released under the MIT license (see COPYING.MIT)
# ./runtest.py -b build_data.json -a tag -f test.manifest

import sys
import os
import time
import unittest
import inspect
from functools import wraps

BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
sys.path.append(os.path.join(BASEDIR, "oeqa"))
sys.path.append(os.path.join(BASEDIR, "bitbake", "lib"))

from optparse import OptionParser
from oeqa.oetest import oeTest
from oeqa.oetest import oeRuntimeTest
from oeqa.oetest import runTests
from oeqa.runexported import FakeTarget
from oeqa.runexported import MyDataDict
from oeqa.runexported import TestContext
from oeqa.utils.sshcontrol import SSHControl
from oeqa.utils.decorators import gettag

try:
    import simplejson as json
except ImportError:
    import json

def tearDown(self):
    pass
oeRuntimeTest.tearDown = tearDown

def wrap_runner(runner, *wargs, **wkwargs):
    @wraps(runner)
    def __wrapper(*args, **kwargs):
        # args and kwargs will overwrite the wargs and wkwargs
        _args = list(args)
        _args.extend(wargs[len(args):] if len(wargs) > len(args) else [])
        kw = wkwargs.copy()
        kw.update(kwargs)
        return runner(*_args, **kw)
    return __wrapper

def main():

    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage)
    parser.add_option("-t", "--target-ip", dest="ip",
            help="The IP address of the target machine. Use this to \
            overwrite the value determined from TEST_TARGET_IP at build time")
    parser.add_option("-s", "--server-ip", dest="server_ip",
            help="The IP address of this machine. Use this to \
            overwrite the value determined from TEST_SERVER_IP at build time.")
    parser.add_option("-d", "--deploy-dir", dest="deploy_dir",
            default=os.path.join(BASEDIR, "deploy"),
            help="Full path to the package feeds, that this \
            the contents of what used to be DEPLOY_DIR on the build machine. \
            If not specified it will use the value specified in the json if \
            that directory actually exists or it will error out.")
    parser.add_option("-l", "--log-dir", dest="log_dir",
            help="This sets the path for TEST_LOG_DIR. If not specified \
            the current dir is used. This is used for usually creating a \
            ssh log file and a scp test file.")
    parser.add_option("-f", "--test-manifest", dest="tests_list",
            help="The test list file")
    parser.add_option("-b", "--build-data", dest="build_data",
            help="The build data file.")
    parser.add_option("-a", "--tag", dest="tag",
            help="The tags to filter test case")
    parser.add_option("-m", "--machine", dest="machine",
            help="""The target machine:quark intel-corei7-64 beaglebone""")
    parser.add_option("-n", "--nativearch", dest="nativearch",
            help="The native arch")
    parser.add_option("-x", "--xunit", dest="xunit",
            help="Output directory to put results in xUnit XML format")

    (options, args) = parser.parse_args()

    tc = TestContext()

    #inject testcase list
    tclist = []
    if not options.tests_list:
        options.tests_list = os.path.join(os.path.dirname(__file__), "testplan", "iottest.manifest")
    for each_manifest in options.tests_list.split():
        with open(each_manifest, "r") as f:
            map(lambda y:tclist.append(y) if y not in tclist else None,
                filter(lambda x: not x.startswith('#'),
                              [n.strip() for n in f.readlines()])
                )
    tc.testslist = tclist
    print tc.testslist

    deployDir = os.path.abspath(options.deploy_dir)
    if not os.path.isdir(deployDir):
        raise Exception("The path to DEPLOY_DIR does not exists: %s" % deployDir)
    if options.machine:
        machine = options.machine
    else:
        parser.error("Please specify target machine by -m")
    if options.xunit:
        try:
            import xmlrunner
        except Exception:
            raise Exception(
              "xUnit output requested but unittest-xml-reporting not installed")
        unittest.TextTestRunner = wrap_runner(xmlrunner.XMLTestRunner, output=options.xunit)
    if options.build_data:
        build_data = options.build_data
    else:
        build_data = os.path.join(deployDir, "files", machine, "builddata.json")
    #get build data from file
    with open(build_data, "r") as f:
        loaded = json.load(f)
    #inject build datastore
    d = MyDataDict()
    if loaded.has_key("d"):
        for key in loaded["d"].keys():
            d[key] = loaded["d"][key]
    d["DEPLOY_DIR"], d["MACHINE"] = deployDir, machine
    if options.log_dir:
        d["TEST_LOG_DIR"] = os.path.abspath(options.log_dir)
    else:
        d["TEST_LOG_DIR"] = os.path.abspath(os.path.dirname(__file__))

    navarch = os.popen("uname -m").read().strip()
    d["BUILD_ARCH"] = "x86_64" if not navarch else navarch
    if options.nativearch:
        d["BUILD_ARCH"] = options.nativearch
    setattr(tc, "d", d)

    #inject build package manifest
    pkgs = [pname.strip() for pname in loaded["pkgmanifest"]]
    setattr(tc, "pkgmanifest", " ".join(pkgs))

    #inject target information
    target = FakeTarget(d)
    target.ip = options.ip if options.ip else "192.168.7.2"
    target.server_ip = options.server_ip if options.server_ip else "192.168.7.1"
    setattr(tc, "target", target)

    #inject others
    for key in loaded.keys():
        if key not in ["testslist", "d", "target", "pkgmanifest"]:
            setattr(tc, key, loaded[key])

    target.exportStart()
    setattr(tc, "tagexp", options.tag)
    runTests(tc)

    return 0

if __name__ == "__main__":
    try:
        ret = main()
    except Exception:
        ret = 1
        import traceback
        traceback.print_exc(5)
    sys.exit(ret)
<commit_msg>runtest: remove a runtime tearDown since upstream had fixed this issue<commit_after>#!/usr/bin/env python

# Copyright (C) 2015 Intel Corporation
#
# Released under the MIT license (see COPYING.MIT)
# ./runtest.py -b build_data.json -a tag -f test.manifest

import sys
import os
import time
import unittest
import inspect
from functools import wraps

BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
sys.path.append(os.path.join(BASEDIR, "oeqa"))
sys.path.append(os.path.join(BASEDIR, "bitbake", "lib"))

from optparse import OptionParser
from oeqa.oetest import oeTest
from oeqa.oetest import oeRuntimeTest
from oeqa.oetest import runTests
from oeqa.runexported import FakeTarget
from oeqa.runexported import MyDataDict
from oeqa.runexported import TestContext
from oeqa.utils.sshcontrol import SSHControl
from oeqa.utils.decorators import gettag

try:
    import simplejson as json
except ImportError:
    import json

def wrap_runner(runner, *wargs, **wkwargs):
    @wraps(runner)
    def __wrapper(*args, **kwargs):
        # args and kwargs will overwrite the wargs and wkwargs
        _args = list(args)
        _args.extend(wargs[len(args):] if len(wargs) > len(args) else [])
        kw = wkwargs.copy()
        kw.update(kwargs)
        return runner(*_args, **kw)
    return __wrapper

def main():

    usage = "usage: %prog [options]"
    parser = OptionParser(usage=usage)
    parser.add_option("-t", "--target-ip", dest="ip",
            help="The IP address of the target machine. Use this to \
            overwrite the value determined from TEST_TARGET_IP at build time")
    parser.add_option("-s", "--server-ip", dest="server_ip",
            help="The IP address of this machine. Use this to \
            overwrite the value determined from TEST_SERVER_IP at build time.")
    parser.add_option("-d", "--deploy-dir", dest="deploy_dir",
            default=os.path.join(BASEDIR, "deploy"),
            help="Full path to the package feeds, that this \
            the contents of what used to be DEPLOY_DIR on the build machine. \
            If not specified it will use the value specified in the json if \
            that directory actually exists or it will error out.")
    parser.add_option("-l", "--log-dir", dest="log_dir",
            help="This sets the path for TEST_LOG_DIR. If not specified \
            the current dir is used. This is used for usually creating a \
            ssh log file and a scp test file.")
    parser.add_option("-f", "--test-manifest", dest="tests_list",
            help="The test list file")
    parser.add_option("-b", "--build-data", dest="build_data",
            help="The build data file.")
    parser.add_option("-a", "--tag", dest="tag",
            help="The tags to filter test case")
    parser.add_option("-m", "--machine", dest="machine",
            help="""The target machine:quark intel-corei7-64 beaglebone""")
    parser.add_option("-n", "--nativearch", dest="nativearch",
            help="The native arch")
    parser.add_option("-x", "--xunit", dest="xunit",
            help="Output directory to put results in xUnit XML format")

    (options, args) = parser.parse_args()

    tc = TestContext()

    #inject testcase list
    tclist = []
    if not options.tests_list:
        options.tests_list = os.path.join(os.path.dirname(__file__), "testplan", "iottest.manifest")
    for each_manifest in options.tests_list.split():
        with open(each_manifest, "r") as f:
            map(lambda y:tclist.append(y) if y not in tclist else None,
                filter(lambda x: not x.startswith('#'),
                              [n.strip() for n in f.readlines()])
                )
    tc.testslist = tclist
    print tc.testslist

    deployDir = os.path.abspath(options.deploy_dir)
    if not os.path.isdir(deployDir):
        raise Exception("The path to DEPLOY_DIR does not exists: %s" % deployDir)
    if options.machine:
        machine = options.machine
    else:
        parser.error("Please specify target machine by -m")
    if options.xunit:
        try:
            import xmlrunner
        except Exception:
            raise Exception(
              "xUnit output requested but unittest-xml-reporting not installed")
        unittest.TextTestRunner = wrap_runner(xmlrunner.XMLTestRunner, output=options.xunit)
    if options.build_data:
        build_data = options.build_data
    else:
        build_data = os.path.join(deployDir, "files", machine, "builddata.json")
    #get build data from file
    with open(build_data, "r") as f:
        loaded = json.load(f)
    #inject build datastore
    d = MyDataDict()
    if loaded.has_key("d"):
        for key in loaded["d"].keys():
            d[key] = loaded["d"][key]
    d["DEPLOY_DIR"], d["MACHINE"] = deployDir, machine
    if options.log_dir:
        d["TEST_LOG_DIR"] = os.path.abspath(options.log_dir)
    else:
        d["TEST_LOG_DIR"] = os.path.abspath(os.path.dirname(__file__))

    navarch = os.popen("uname -m").read().strip()
    d["BUILD_ARCH"] = "x86_64" if not navarch else navarch
    if options.nativearch:
        d["BUILD_ARCH"] = options.nativearch
    setattr(tc, "d", d)

    #inject build package manifest
    pkgs = [pname.strip() for pname in loaded["pkgmanifest"]]
    setattr(tc, "pkgmanifest", " ".join(pkgs))

    #inject target information
    target = FakeTarget(d)
    target.ip = options.ip if options.ip else "192.168.7.2"
    target.server_ip = options.server_ip if options.server_ip else "192.168.7.1"
    setattr(tc, "target", target)

    #inject others
    for key in loaded.keys():
        if key not in ["testslist", "d", "target", "pkgmanifest"]:
            setattr(tc, key, loaded[key])

    target.exportStart()
    setattr(tc, "tagexp", options.tag)
    runTests(tc)

    return 0

if __name__ == "__main__":
    try:
        ret = main()
    except Exception:
        ret = 1
        import traceback
        traceback.print_exc(5)
    sys.exit(ret)
<|endoftext|>