<commit_before>import os.path
import sys
import warnings
import unittest

test_modules = [
    'cryptutil',
    'oidutil',
    'dh',
    ]

def fixpath():
    try:
        d = os.path.dirname(__file__)
    except NameError:
        d = os.path.dirname(sys.argv[0])
    parent = os.path.normpath(os.path.join(d, '..'))
    if parent not in sys.path:
        print(("putting %s in sys.path" % (parent,)))
        sys.path.insert(0, parent)

def otherTests():
    suite = unittest.TestSuite()
    for module_name in test_modules:
        module_name = 'openid.test.' + module_name
        try:
            test_mod = __import__(module_name, {}, {}, [None])
        except ImportError:
            print(('Failed to import test %r' % (module_name,)))
        else:
            suite.addTest(unittest.FunctionTestCase(test_mod.test))

    return suite

def pyUnitTests():
    pyunit_module_names = [
        'server',
        'consumer',
        'message',
        'symbol',
        'etxrd',
        'xri',
        'xrires',
        'association_response',
        'auth_request',
        'negotiation',
        'verifydisco',
        'sreg',
        'ax',
        'pape',
        'pape_draft2',
        'pape_draft5',
        'rpverify',
        'extension',
        ]

    pyunit_modules = [
        __import__('openid.test.test_%s' % (name,), {}, {}, ['unused'])
        for name in pyunit_module_names
        ]

    try:
        from openid.test import test_examples
    except ImportError:
        e = sys.exc_info()[1]
        if 'twill' in str(e):
            warnings.warn("Could not import twill; skipping test_examples.")
        else:
            raise
    else:
        pyunit_modules.append(test_examples)

    # Some modules have data-driven tests, and they use custom methods
    # to build the test suite:
    custom_module_names = [
        'kvform',
        'linkparse',
        'oidutil',
        'storetest',
        'test_accept',
        'test_association',
        'test_discover',
        'test_fetchers',
        'test_htmldiscover',
        'test_nonce',
        'test_openidyadis',
        'test_parsehtml',
        'test_urinorm',
        'test_yadis_discover',
        'trustroot',
        ]

    loader = unittest.TestLoader()
    s = unittest.TestSuite()

    for m in pyunit_modules:
        s.addTest(loader.loadTestsFromModule(m))

    for name in custom_module_names:
        m = __import__('openid.test.%s' % (name,), {}, {}, ['unused'])
        try:
            s.addTest(m.pyUnitTests())
        except AttributeError:
            # because the AttributeError doesn't actually say which
            # object it was.
            print(("Error loading tests from %s:" % (name,)))
            raise

    return s

def splitDir(d, count):
    # in python2.4 and above, it's easier to spell this as
    # d.rsplit(os.sep, count)
    for i in range(count):
        d = os.path.dirname(d)
    return d

def _import_djopenid():
    """Import djopenid from examples/

    It's not in sys.path, and I don't really want to put it in sys.path.
    """
    import types
    thisfile = os.path.abspath(sys.modules[__name__].__file__)
    topDir = splitDir(thisfile, 2)
    djdir = os.path.join(topDir, 'examples', 'djopenid')

    djinit = os.path.join(djdir, '__init__.py')

    djopenid = types.ModuleType('djopenid')
    if sys.version_info[0] >= 3:
        with open(djinit, 'r') as f:
            exec(compile(f.read(), "__init__.py"), "exec", djinit.__dict__)
    else:
        exec(compile(open(djinit).read(), djinit, 'exec'), djopenid.__dict__)
    djopenid.__file__ = djinit

    # __path__ is the magic that makes child modules of the djopenid package
    # importable.  New feature in python 2.3, see PEP 302.
    djopenid.__path__ = [djdir]
    sys.modules['djopenid'] = djopenid

def django_tests():
    """Runs tests from examples/djopenid.

    @returns: number of failed tests.
    """
    import os
    # Django uses this to find out where its settings are.
    os.environ['DJANGO_SETTINGS_MODULE'] = 'djopenid.settings'

    _import_djopenid()

    try:
        import django.test.simple
    except ImportError:
        warnings.warn("django.test.simple not found; "
                      "django examples not tested.")
        return 0

    import djopenid.server.models, djopenid.consumer.models
    print ("Testing Django examples:")

    # These tests do get put in to a pyunit test suite, so we could run them
    # with the other pyunit tests, but django also establishes a test database
    # for them, so we let it do that thing instead.
    return django.test.simple.run_tests([djopenid.server.models,
                                         djopenid.consumer.models])

try:
    bool
except NameError:
    def bool(x):
        return not not x

def test_suite():
    fixpath()
    all_suite = unittest.TestSuite()
    all_suite.addTests(otherTests())
    all_suite.addTests(pyUnitTests())
    all_suite.addTest(unittest.FunctionTestCase(django_tests))
    return all_suite
<commit_msg>Clean up import & fix test error<commit_after>import os.path
import sys
import warnings
import unittest

test_modules = [
    'cryptutil',
    'oidutil',
    'dh',
    ]

def fixpath():
    try:
        d = os.path.dirname(__file__)
    except NameError:
        d = os.path.dirname(sys.argv[0])
    parent = os.path.normpath(os.path.join(d, '..'))
    if parent not in sys.path:
        print(("putting %s in sys.path" % (parent,)))
        sys.path.insert(0, parent)

def otherTests():
    suite = unittest.TestSuite()
    for module_name in test_modules:
        module_name = 'openid.test.' + module_name
        try:
            test_mod = __import__(module_name, {}, {}, [None])
        except ImportError:
            print(('Failed to import test %r' % (module_name,)))
        else:
            suite.addTest(unittest.FunctionTestCase(test_mod.test))

    return suite

def pyUnitTests():
    pyunit_module_names = [
        'server',
        'consumer',
        'message',
        'symbol',
        'etxrd',
        'xri',
        'xrires',
        'association_response',
        'auth_request',
        'negotiation',
        'verifydisco',
        'sreg',
        'ax',
        'pape',
        'pape_draft2',
        'pape_draft5',
        'rpverify',
        'extension',
        ]

    pyunit_modules = [
        __import__('openid.test.test_%s' % (name,), {}, {}, ['unused'])
        for name in pyunit_module_names
        ]

    try:
        from openid.test import test_examples
    except ImportError:
        e = sys.exc_info()[1]
        if 'twill' in str(e):
            warnings.warn("Could not import twill; skipping test_examples.")
        else:
            raise
    else:
        pyunit_modules.append(test_examples)

    # Some modules have data-driven tests, and they use custom methods
    # to build the test suite:
    custom_module_names = [
        'kvform',
        'linkparse',
        'oidutil',
        'storetest',
        'test_accept',
        'test_association',
        'test_discover',
        'test_fetchers',
        'test_htmldiscover',
        'test_nonce',
        'test_openidyadis',
        'test_parsehtml',
        'test_urinorm',
        'test_yadis_discover',
        'trustroot',
        ]

    loader = unittest.TestLoader()
    s = unittest.TestSuite()

    for m in pyunit_modules:
        s.addTest(loader.loadTestsFromModule(m))

    for name in custom_module_names:
        m = __import__('openid.test.%s' % (name,), {}, {}, ['unused'])
        try:
            s.addTest(m.pyUnitTests())
        except AttributeError:
            # because the AttributeError doesn't actually say which
            # object it was.
            print(("Error loading tests from %s:" % (name,)))
            raise

    return s

def _import_djopenid():
    """Import djopenid from examples/

    It's not in sys.path, and I don't really want to put it in sys.path.
    """
    import types
    parentDir = os.path.join(__file__, "..", "..")
    topDir = os.path.abspath(os.path.join(parentDir, ".."))
    djdir = os.path.join(topDir, 'examples', 'djopenid')

    djinit = os.path.join(djdir, '__init__.py')

    djopenid = types.ModuleType('djopenid')
    with open(djinit, 'r') as f:
        exec(compile(f.read(), "__init__.py", "exec"), {})

    djopenid.__file__ = djinit

    # __path__ is the magic that makes child modules of the djopenid package
    # importable.  New feature in python 2.3, see PEP 302.
    djopenid.__path__ = [djdir]
    sys.modules['djopenid'] = djopenid

def django_tests():
    """Runs tests from examples/djopenid.

    @returns: number of failed tests.
    """
    import os
    # Django uses this to find out where its settings are.
    os.environ['DJANGO_SETTINGS_MODULE'] = 'djopenid.settings'

    _import_djopenid()

    try:
        import django.test.simple
    except ImportError:
        warnings.warn("django.test.simple not found; "
                      "django examples not tested.")
        return 0

    import djopenid.server.models, djopenid.consumer.models
    print ("Testing Django examples:")

    # These tests do get put in to a pyunit test suite, so we could run them
    # with the other pyunit tests, but django also establishes a test database
    # for them, so we let it do that thing instead.
    return django.test.simple.run_tests([djopenid.server.models,
                                         djopenid.consumer.models])

try:
    bool
except NameError:
    def bool(x):
        return not not x

def test_suite():
    fixpath()
    all_suite = unittest.TestSuite()
    all_suite.addTests(otherTests())
    all_suite.addTests(pyUnitTests())
    all_suite.addTest(unittest.FunctionTestCase(django_tests))
    return all_suite
<|endoftext|>


<commit_before>#! /usr/bin/python2.3
# vim:sw=8:ts=8:et:nowrap

import sys
import re
import os
import string

import mx.DateTime

from miscfuncs import ApplyFixSubstitutions

# this filter converts column number tags of form:
#     9 Dec 2003 : Column 893
# into xml form
#     <stamp coldate="2003-12-09" colnum="893"/>

# this accounts for the cases where the colnum at the very start is left out.
fixsubs = 	[
	( '(
THE PARLIAMENTARY DEBATES
)', '
\n\n14 Oct 2003 : Column 1
\n\\1', 1, '2003-10-14'),
	( '(
THE PARLIAMENTARY DEBATES
)', '
\n\n14 Jul 2003 : Column 1
\n\\1', 1, '2003-07-14'),

	( '(
\n
)(\s*23 Jun 2003 : Column 836
)', '\\1
\\2', 1, '2003-06-23'),
	( '27 Mar 2003 : Column 563
\s*

\s*

', '', 1, '2003-03-27'),
	( '10 Mar 2003 : Column 141
\s*

\s*

', '', 1, '2003-03-10'),
	( '4 Feb 2003 : Column 251
\s*

\s*

', '', 1, '2003-02-04'),
	( '24 Sept 2002 : Column 155
\s*

\s*

', '', 1, '2002-09-24'),
	( '
2.20
', '
2.20 pm
', 1, '2003-02-28'),
        ( '(
2.58)(
)', '\\1 pm\\2', 1, '2004-01-13'),

        ( '30 Apr 2003 : Column 401
\s*

\s*

', '', 1, '2003-04-30'),
        ( '6 May 2003 : Column 671
\s*

\s*

', '', 1, '2003-05-06'),

        ( '(fine, or both.".\'\&\#151\;)', '\\1

', 1, '2003-05-20'),
        ( '(	"Section)', '
\\1', 1, '2003-05-20'),

        ( '()( 8. )(Mr. Alistair Carmichael)', '\\2\\1\\3', 1, '2003-01-07'),

        ( '( As a number of Members have said, .*?)', '\\1', 1, '2003-06-20'),
]

# 9 Dec 2003 : Column 893
regcolumnum1 = '
\s*[^:<]*:\s*column\s*\d+\s*
\n(?i)'
regcolumnum2 = '
\s*
\s*[^:<]*:\s*column\s*\d+\s*
\n
(?i)'
regcolumnum3 = '
\s*
\s*[^:<]*:\s*column\s*\d+\s*
\n
]*>(?i)'
regcolumnum4 = '
\s*\s*[^:<]*:\s*column\s*\d+\s*
\n]*>(?i)'
recolumnumvals = re.compile('(?:
||
||\s)*([^:<]*):\s*column\s*(\d+)\s*(?:
|
|]*>|\s)*$(?i)')

#13 Nov 2003 : Column 431—continued
# these occur infrequently
regcolnumcont = '[^:<]*:\s*column\s*\d+—continued(?i)'
recolnumcontvals = re.compile('([^:<]*):\s*column\s*(\d+)—continued(?i)')

#
12.31 pm

#
\n12.31 pm\n

# [3:31 pm
    -- at the beginning of divisions
regtime = '(?:</?p>\s*|<h[45]>|\[|\n)(?:\d+(?:[:\.]\d+)?\s*[ap]\.?m\.?(?:</st>)?|12 noon)(?:\s*</?p>|\s*</h[45]>|\n)(?i)'
retimevals = re.compile('(?:</?p>\s*|<h\d>|\[|\n)\s*(\d+(?:[:\.]\d+)?\s*[apmnon.]+)(?i)')

#
reaname = '(?i)'
reanamevals = re.compile('(?i)')

recomb = re.compile('(%s|%s|%s|%s|%s|%s|%s)' % (regcolumnum1, regcolumnum2, regcolumnum3, regcolumnum4, regcolnumcont, regtime, reaname))
remarginal = re.compile(':\s*column\s*(\d+)|\n(?:\d+[.:])?\d+\s*[ap]\.?m\.?[^,\w](?i)|</?a[\s>]')

# This one used to break times into component parts: 7.10 pm
regparsetime = re.compile("^(\d+)[\.:](\d+)\s?([\w\.]+)$")
# 7 pm
regparsetimeonhour = re.compile("^(\d+)()\s?([\w\.]+)$")

def FilterDebateColTime(fout, text, sdate):
	text = ApplyFixSubstitutions(text, sdate, fixsubs)

	colnum = -1
	time = ''	# need to use a proper timestamp code class
	#fs =
	for fss in recomb.split(text):
#                print "fss=", fss
#                print "######################################"
		# column number type
		columng = recolumnumvals.match(fss)
		if columng:
			# check date
			ldate = mx.DateTime.DateTimeFrom(columng.group(1)).date
			if sdate != ldate:
				raise Exception, "Column date disagrees %s -- %s" % (sdate, fss)

			# check number
			lcolnum = string.atoi(columng.group(2))
			if lcolnum == colnum - 1:
				pass	# spurious decrementing of column number stamps
			elif (colnum == -1) or (lcolnum == colnum + 1):
				pass  # good
			# column numbers do get skipped during division listings
			elif lcolnum < colnum:
				raise Exception, "Colnum not incrementing %d -- %s" % (lcolnum, fss)

			# write a column number stamp
			colnum = lcolnum
			fout.write('<stamp coldate="%s" colnum="%s"/>' % (sdate, colnum))
			continue

		columncg = recolnumcontvals.match(fss)
		if columncg:
			ldate = mx.DateTime.DateTimeFrom(columncg.group(1)).date
			if sdate != ldate:
				raise Exception, "Column date disagrees %s -- %s" % (sdate, fss)

			lcolnum = string.atoi(columncg.group(2))
			if colnum != lcolnum:
				raise Exception, "Cont column number disagrees %d -- %s" % (colnum, fss)

			continue

		timeg = retimevals.match(fss)
		if timeg:
			time = timeg.group(1)
                        #print "time ", time

			# This code lifted from fix_time PHP code from easyParliament
			# (thanks Phil!)
			timeparts = regparsetime.match(time)
			if not timeparts:
			    timeparts = regparsetimeonhour.match(time)
			if timeparts:
			    hour = int(timeparts.group(1))
			    if (timeparts.group(2) <> ""):
				mins = int(timeparts.group(2))
			    else:
				mins = 0
			    meridien = timeparts.group(3)

			    if (meridien == 'pm' or meridien == 'p.m.') and hour <> 12:
				hour += 12

			    time = "%02d:%02d:00" % (hour, mins)
			else:
			    time = "unknown " + time
			    raise Exception, "Time not matched: " + time

			fout.write('<stamp time="%s"/>' % time)
			continue

                # anchor names from HTML
                anameg = reanamevals.match(fss)
                if anameg:
                        aname = anameg.group(1)
                        fout.write('<stamp aname="%s"/>' % aname)
                        continue

		# nothing detected
		# check if we've missed anything obvious
		if recomb.match(fss):
			print fss
			raise Exception, ' regexpvals not general enough '
		if remarginal.search(fss):
			print ' marginal coltime detection case '
			print remarginal.search(fss).groups()
			print remarginal.search(fss).group(0)
                        print '--------------------------------\n'
			print fss
			sys.exit()
		fout.write(fss)

<commit_msg>Error<commit_after>#! /usr/bin/python2.3
# vim:sw=8:ts=8:et:nowrap

import sys
import re
import os
import string

import mx.DateTime

from miscfuncs import ApplyFixSubstitutions

# this filter converts column number tags of form:
#     9 Dec 2003 : Column 893
# into xml form
#     <stamp coldate="2003-12-09" colnum="893"/>

# this accounts for the cases where the colnum at the very start is left out.
fixsubs = 	[
	( '(
THE PARLIAMENTARY DEBATES
)', '
\n\n14 Oct 2003 : Column 1
\n\\1', 1, '2003-10-14'),
	( '(
THE PARLIAMENTARY DEBATES
)', '
\n\n14 Jul 2003 : Column 1
\n\\1', 1, '2003-07-14'),

	( '(
\n
)(\s*23 Jun 2003 : Column 836
)', '\\1
\\2', 1, '2003-06-23'),
	( '27 Mar 2003 : Column 563
\s*

\s*

', '', 1, '2003-03-27'),
	( '10 Mar 2003 : Column 141
\s*

\s*

', '', 1, '2003-03-10'),
	( '4 Feb 2003 : Column 251
\s*

\s*

', '', 1, '2003-02-04'),
	( '24 Sept 2002 : Column 155
\s*

\s*

', '', 1, '2002-09-24'),
	( '
2.20
', '
2.20 pm
', 1, '2003-02-28'),
        ( '(
2.58)(
)', '\\1 pm\\2', 1, '2004-01-13'),

        ( '30 Apr 2003 : Column 401
\s*

\s*

', '', 1, '2003-04-30'),
        ( '6 May 2003 : Column 671
\s*

\s*

', '', 1, '2003-05-06'),

        ( '(fine, or both.".\'\&\#151\;)', '\\1

', 1, '2003-05-20'),
        ( '(	"Section)', '
\\1', 1, '2003-05-20'),

        ( '()( 8. )(Mr. Alistair Carmichael)', '\\2\\1\\3', 1, '2003-01-07'),

        ( '( As a number of Members have said, .*?)', '\\1', 1, '2003-06-20'),

        # Column numbers are totally knackered here
        ( '16 Jun 2003 : Column 184
', '', 1, '2003-06-16'),
]

# 9 Dec 2003 : Column 893
regcolumnum1 = '
\s*[^:<]*:\s*column\s*\d+\s*
\n(?i)'
regcolumnum2 = '
\s*
\s*[^:<]*:\s*column\s*\d+\s*
\n
(?i)'
regcolumnum3 = '
\s*
\s*[^:<]*:\s*column\s*\d+\s*
\n
]*>(?i)'
regcolumnum4 = '
\s*\s*[^:<]*:\s*column\s*\d+\s*
\n]*>(?i)'
recolumnumvals = re.compile('(?:
||
||\s)*([^:<]*):\s*column\s*(\d+)\s*(?:
|
|]*>|\s)*$(?i)')

#13 Nov 2003 : Column 431—continued
# these occur infrequently
regcolnumcont = '[^:<]*:\s*column\s*\d+—continued(?i)'
recolnumcontvals = re.compile('([^:<]*):\s*column\s*(\d+)—continued(?i)')

#
12.31 pm

#
\n12.31 pm\n

# [3:31 pm
    -- at the beginning of divisions
regtime = '(?:</?p>\s*|<h[45]>|\[|\n)(?:\d+(?:[:\.]\d+)?\s*[ap]\.?m\.?(?:</st>)?|12 noon)(?:\s*</?p>|\s*</h[45]>|\n)(?i)'
retimevals = re.compile('(?:</?p>\s*|<h\d>|\[|\n)\s*(\d+(?:[:\.]\d+)?\s*[apmnon.]+)(?i)')

#
reaname = '(?i)'
reanamevals = re.compile('(?i)')

recomb = re.compile('(%s|%s|%s|%s|%s|%s|%s)' % (regcolumnum1, regcolumnum2, regcolumnum3, regcolumnum4, regcolnumcont, regtime, reaname))
remarginal = re.compile(':\s*column\s*(\d+)|\n(?:\d+[.:])?\d+\s*[ap]\.?m\.?[^,\w](?i)|</?a[\s>]')

# This one used to break times into component parts: 7.10 pm
regparsetime = re.compile("^(\d+)[\.:](\d+)\s?([\w\.]+)$")
# 7 pm
regparsetimeonhour = re.compile("^(\d+)()\s?([\w\.]+)$")

def FilterDebateColTime(fout, text, sdate):
	text = ApplyFixSubstitutions(text, sdate, fixsubs)

	colnum = -1
	time = ''	# need to use a proper timestamp code class
	#fs =
	for fss in recomb.split(text):
#                print "fss=", fss
#                print "######################################"
		# column number type
		columng = recolumnumvals.match(fss)
		if columng:
			# check date
			ldate = mx.DateTime.DateTimeFrom(columng.group(1)).date
			if sdate != ldate:
				raise Exception, "Column date disagrees %s -- %s" % (sdate, fss)

			# check number
			lcolnum = string.atoi(columng.group(2))
			if lcolnum == colnum - 1:
				pass	# spurious decrementing of column number stamps
			elif (colnum == -1) or (lcolnum == colnum + 1):
				pass  # good
			# column numbers do get skipped during division listings
			elif lcolnum < colnum:
				raise Exception, "Colnum not incrementing %d -- %s" % (lcolnum, fss)

			# write a column number stamp
			colnum = lcolnum
			fout.write('<stamp coldate="%s" colnum="%s"/>' % (sdate, colnum))
			continue

		columncg = recolnumcontvals.match(fss)
		if columncg:
			ldate = mx.DateTime.DateTimeFrom(columncg.group(1)).date
			if sdate != ldate:
				raise Exception, "Column date disagrees %s -- %s" % (sdate, fss)

			lcolnum = string.atoi(columncg.group(2))
			if colnum != lcolnum:
				raise Exception, "Cont column number disagrees %d -- %s" % (colnum, fss)

			continue

		timeg = retimevals.match(fss)
		if timeg:
			time = timeg.group(1)
                        #print "time ", time

			# This code lifted from fix_time PHP code from easyParliament
			# (thanks Phil!)
			timeparts = regparsetime.match(time)
			if not timeparts:
			    timeparts = regparsetimeonhour.match(time)
			if timeparts:
			    hour = int(timeparts.group(1))
			    if (timeparts.group(2) <> ""):
				mins = int(timeparts.group(2))
			    else:
				mins = 0
			    meridien = timeparts.group(3)

			    if (meridien == 'pm' or meridien == 'p.m.') and hour <> 12:
				hour += 12

			    time = "%02d:%02d:00" % (hour, mins)
			else:
			    time = "unknown " + time
			    raise Exception, "Time not matched: " + time

			fout.write('<stamp time="%s"/>' % time)
			continue

                # anchor names from HTML
                anameg = reanamevals.match(fss)
                if anameg:
                        aname = anameg.group(1)
                        fout.write('<stamp aname="%s"/>' % aname)
                        continue

		# nothing detected
		# check if we've missed anything obvious
		if recomb.match(fss):
			print fss
			raise Exception, ' regexpvals not general enough '
		if remarginal.search(fss):
			print ' marginal coltime detection case '
			print remarginal.search(fss).groups()
			print remarginal.search(fss).group(0)
                        print '--------------------------------\n'
			print fss
			sys.exit()
		fout.write(fss)

<|endoftext|>


<commit_before>import imaplib
from flanker import mime
import collections

mail = imaplib.IMAP4_SSL('imap.gmail.com')
mail.login('Your Email', 'Your Password')

# mail.select("inbox") # connect to inbox.
mail.select("[Gmail]/All Mail")

result, data = mail.search(None, "ALL")

ids = data[0]
id_list = ids.split()

c = collections.Counter()
for email_id in id_list:
	# for email_id in id_list[-1000:]:
	result, data = mail.fetch(email_id, "(RFC822)")
	message_string = data[0][1]
	msg = mime.from_string(message_string)
	sender = [msg.headers.items()[i][1] for i, item in enumerate(msg.headers.items()) if item[0] == 'From']
	print sender
	c.update(sender)

fo = open("inbound_emails.txt", "wb")
for item in c:
	fo.write("{}\t{}\n".format(item, c[item]));

fo.close()
<commit_msg>MAJOR REFACTOR<commit_after>import imaplib
from flanker import mime
import collections

mail = imaplib.IMAP4_SSL('imap.gmail.com')
mail.login('Your Email', 'Your Password')

# mail.select("inbox") # connect to inbox.
mail.select("[Gmail]/All Mail")

result, data = mail.search(None, "ALL")

id_list = data[0].split()

c = collections.Counter()
for email_id in id_list:
	# for email_id in id_list[-1000:]:
	result, data = mail.fetch(email_id, "(RFC822)")
	message_string = data[0][1]
	msg = mime.from_string(message_string)
	sender = [msg.headers.items()[i][1] for i, item in enumerate(msg.headers.items()) if item[0] == 'From']
	print sender
	c.update(sender)

fo = open("inbound_emails.txt", "wb")
for item in c:
	fo.write("{}\t{}\n".format(item, c[item]));

fo.close()
<|endoftext|>


<commit_before>from AlphaGo.preprocessing.preprocessing import Preprocess
import AlphaGo.go as go
import numpy as np
import unittest

def simple_board():
	# make a tiny board for the sake of testing and hand-coding expected results
	#
	#         X
	#   0 1 2 3 4 5 6
	#   B W . . . . . 0
	#   B W . . . . . 1
	#   B . . . B . . 2
	# Y . . . B k B . 3
	#   . . . W B W . 4
	#   . . . . W . . 5
	#   . . . . . . . 6
	#
	# where k is a ko position (white was just captured)

	gs = go.GameState(size=7)

	# ladder-looking thing in the top-left
	gs.do_move((0, 0))  # B
	gs.do_move((1, 0))  # W
	gs.do_move((0, 1))  # B
	gs.do_move((1, 1))  # W
	gs.do_move((0, 2))  # B

	# ko position in the middle
	gs.do_move((3, 4))  # W
	gs.do_move((3, 3))  # B
	gs.do_move((4, 5))  # W
	gs.do_move((4, 2))  # B
	gs.do_move((5, 4))  # W
	gs.do_move((5, 3))  # B
	gs.do_move((4, 3))  # W - the ko position
	gs.do_move((4, 4))  # B - does the capture

	return gs

class TestPreprocessingFeatures(unittest.TestCase):
	"""Test the functions in preprocessing.py

	note that the hand-coded features look backwards from what is depicted
	in simple_board() because of the x/y column/row transpose thing (i.e.
	numpy is typically thought of as indexing rows first, but we use (x,y)
	indexes, so a numpy row is like a go column and vice versa)
	"""

	def test_get_board(self):
		gs = simple_board()
		pp = Preprocess(["board"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		white_pos = np.asarray([
			[0, 0, 0, 0, 0, 0, 0],
			[1, 1, 0, 0, 0, 0, 0],
			[0, 0, 0, 0, 0, 0, 0],
			[0, 0, 0, 0, 1, 0, 0],
			[0, 0, 0, 0, 0, 1, 0],
			[0, 0, 0, 0, 1, 0, 0],
			[0, 0, 0, 0, 0, 0, 0]])
		black_pos = np.asarray([
			[1, 1, 1, 0, 0, 0, 0],
			[0, 0, 0, 0, 0, 0, 0],
			[0, 0, 0, 0, 0, 0, 0],
			[0, 0, 0, 1, 0, 0, 0],
			[0, 0, 1, 0, 1, 0, 0],
			[0, 0, 0, 1, 0, 0, 0],
			[0, 0, 0, 0, 0, 0, 0]])
		empty_pos = np.ones((gs.size, gs.size)) - (white_pos + black_pos)

		# check number of planes
		self.assertEqual(feature.shape, (gs.size, gs.size, 3))
		# check return value against hand-coded expectation
		# (given that current_player is white)
		self.assertTrue(np.all(feature == np.dstack((white_pos, black_pos, empty_pos))))

	def test_get_turns_since(self):
		gs = simple_board()
		pp = Preprocess(["turns_since"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		one_hot_turns = np.zeros((gs.size, gs.size, 8))
		rev_history = gs.history[::-1]
		# one plane per move for the last 7
		for i in range(7):
			move = rev_history[i]
			one_hot_turns[move[0], move[1], i] = 1
		# far back plane gets all other moves
		for move in rev_history[7:]:
			one_hot_turns[move[0], move[1], 7] = 1

		self.assertTrue(np.all(feature == one_hot_turns))

	def test_get_liberties(self):
		gs = simple_board()
		pp = Preprocess(["liberties"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		# todo - test liberties when > 8

		one_hot_liberties = np.zeros((gs.size, gs.size, 8))
		# black piece at (4,4) has a single liberty: (4,3)
		one_hot_liberties[4, 4, 0] = 1

		# the black group in the top left corner has 2 liberties
		one_hot_liberties[0, 0:3, 1] = 1
		# 	.. as do the white pieces on the left and right of the eye
		one_hot_liberties[3, 4, 1] = 1
		one_hot_liberties[5, 4, 1] = 1

		# the white group in the top left corner has 3 liberties
		one_hot_liberties[1, 0:2, 2] = 1
		# 	...as does the white piece at (4,5)
		one_hot_liberties[4, 5, 2] = 1
		# 	...and the black pieces on the sides of the eye
		one_hot_liberties[3, 3, 2] = 1
		one_hot_liberties[5, 3, 2] = 1

		# the black piece at (4,2) has 4 liberties
		one_hot_liberties[4, 2, 3] = 1

		for i in range(8):
			self.assertTrue(
				np.all(feature[:, :, i] == one_hot_liberties[:, :, i]),
				"bad expectation: stones with %d liberties" % (i + 1))

	def test_get_capture_size(self):
		# TODO - at the moment there is no imminent capture
		gs = simple_board()
		pp = Preprocess(["capture_size"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		one_hot_capture = np.zeros((gs.size, gs.size, 8))
		# there is no capture available; all legal moves are zero-capture
		for (x, y) in gs.get_legal_moves():
			one_hot_capture[x, y, 0] = 1

		for i in range(8):
			self.assertTrue(
				np.all(feature[:, :, i] == one_hot_capture[:, :, i]),
				"bad expectation: capturing %d stones" % i)

	def test_get_self_atari_size(self):
		# TODO - at the moment there is no imminent self-atari for white
		gs = simple_board()
		pp = Preprocess(["self_atari_size"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		self.assertTrue(np.all(feature == np.zeros((gs.size, gs.size, 8))))

	def test_get_liberties_after(self):
		gs = simple_board()
		pp = Preprocess(["liberties_after"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		one_hot_liberties = np.zeros((gs.size, gs.size, 8))

		# TODO (?) hand-code?
		for (x, y) in gs.get_legal_moves():
			copy = gs.copy()
			copy.do_move((x, y))
			libs = copy.liberty_counts[x, y]
			if libs < 7:
				one_hot_liberties[x, y, libs] = 1
			else:
				one_hot_liberties[x, y, 7] = 1

		for i in range(8):
			self.assertTrue(
				np.all(feature[:, :, i] == one_hot_liberties[:, :, i]),
				"bad expectation: stones with %d liberties after move" % (i + 1))

	def test_get_ladder_capture(self):
		pass

	def test_get_ladder_escape(self):
		pass

	def test_get_sensibleness(self):
		# TODO - there are no legal eyes at the moment

		gs = simple_board()
		pp = Preprocess(["sensibleness"])
		feature = pp.state_to_tensor(gs)[0, 0]  # 1D tensor; no need to transpose

		expectation = np.zeros((gs.size, gs.size))
		for (x, y) in gs.get_legal_moves():
			if not (gs.is_eye((x, y), go.WHITE)):
				expectation[x, y] = 1
		self.assertTrue(np.all(expectation == feature))

	def test_feature_concatenation(self):
		gs = simple_board()
		pp = Preprocess(["board", "sensibleness", "capture_size"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		expectation = np.zeros((gs.size, gs.size, 3 + 1 + 8))

		# first three planes: board
		expectation[:, :, 0] = (gs.board == go.WHITE) * 1
		expectation[:, :, 1] = (gs.board == go.BLACK) * 1
		expectation[:, :, 2] = (gs.board == go.EMPTY) * 1

		# 4th plane: sensibleness (as in test_get_sensibleness)
		for (x, y) in gs.get_legal_moves():
			if not (gs.is_eye((x, y), go.WHITE)):
				expectation[x, y, 3] = 1

		# 5th through 12th plane: capture size (all zero-capture)
		for (x, y) in gs.get_legal_moves():
			expectation[x, y, 4] = 1

		self.assertTrue(np.all(expectation == feature))

if __name__ == '__main__':
	unittest.main()
<commit_msg>fixed E303 too many blank lines<commit_after>from AlphaGo.preprocessing.preprocessing import Preprocess
import AlphaGo.go as go
import numpy as np
import unittest

def simple_board():
	# make a tiny board for the sake of testing and hand-coding expected results
	#
	#         X
	#   0 1 2 3 4 5 6
	#   B W . . . . . 0
	#   B W . . . . . 1
	#   B . . . B . . 2
	# Y . . . B k B . 3
	#   . . . W B W . 4
	#   . . . . W . . 5
	#   . . . . . . . 6
	#
	# where k is a ko position (white was just captured)

	gs = go.GameState(size=7)

	# ladder-looking thing in the top-left
	gs.do_move((0, 0))  # B
	gs.do_move((1, 0))  # W
	gs.do_move((0, 1))  # B
	gs.do_move((1, 1))  # W
	gs.do_move((0, 2))  # B

	# ko position in the middle
	gs.do_move((3, 4))  # W
	gs.do_move((3, 3))  # B
	gs.do_move((4, 5))  # W
	gs.do_move((4, 2))  # B
	gs.do_move((5, 4))  # W
	gs.do_move((5, 3))  # B
	gs.do_move((4, 3))  # W - the ko position
	gs.do_move((4, 4))  # B - does the capture

	return gs

class TestPreprocessingFeatures(unittest.TestCase):
	"""Test the functions in preprocessing.py

	note that the hand-coded features look backwards from what is depicted
	in simple_board() because of the x/y column/row transpose thing (i.e.
	numpy is typically thought of as indexing rows first, but we use (x,y)
	indexes, so a numpy row is like a go column and vice versa)
	"""

	def test_get_board(self):
		gs = simple_board()
		pp = Preprocess(["board"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		white_pos = np.asarray([
			[0, 0, 0, 0, 0, 0, 0],
			[1, 1, 0, 0, 0, 0, 0],
			[0, 0, 0, 0, 0, 0, 0],
			[0, 0, 0, 0, 1, 0, 0],
			[0, 0, 0, 0, 0, 1, 0],
			[0, 0, 0, 0, 1, 0, 0],
			[0, 0, 0, 0, 0, 0, 0]])
		black_pos = np.asarray([
			[1, 1, 1, 0, 0, 0, 0],
			[0, 0, 0, 0, 0, 0, 0],
			[0, 0, 0, 0, 0, 0, 0],
			[0, 0, 0, 1, 0, 0, 0],
			[0, 0, 1, 0, 1, 0, 0],
			[0, 0, 0, 1, 0, 0, 0],
			[0, 0, 0, 0, 0, 0, 0]])
		empty_pos = np.ones((gs.size, gs.size)) - (white_pos + black_pos)

		# check number of planes
		self.assertEqual(feature.shape, (gs.size, gs.size, 3))
		# check return value against hand-coded expectation
		# (given that current_player is white)
		self.assertTrue(np.all(feature == np.dstack((white_pos, black_pos, empty_pos))))

	def test_get_turns_since(self):
		gs = simple_board()
		pp = Preprocess(["turns_since"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		one_hot_turns = np.zeros((gs.size, gs.size, 8))
		rev_history = gs.history[::-1]
		# one plane per move for the last 7
		for i in range(7):
			move = rev_history[i]
			one_hot_turns[move[0], move[1], i] = 1
		# far back plane gets all other moves
		for move in rev_history[7:]:
			one_hot_turns[move[0], move[1], 7] = 1

		self.assertTrue(np.all(feature == one_hot_turns))

	def test_get_liberties(self):
		gs = simple_board()
		pp = Preprocess(["liberties"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		# todo - test liberties when > 8

		one_hot_liberties = np.zeros((gs.size, gs.size, 8))
		# black piece at (4,4) has a single liberty: (4,3)
		one_hot_liberties[4, 4, 0] = 1

		# the black group in the top left corner has 2 liberties
		one_hot_liberties[0, 0:3, 1] = 1
		# 	.. as do the white pieces on the left and right of the eye
		one_hot_liberties[3, 4, 1] = 1
		one_hot_liberties[5, 4, 1] = 1

		# the white group in the top left corner has 3 liberties
		one_hot_liberties[1, 0:2, 2] = 1
		# 	...as does the white piece at (4,5)
		one_hot_liberties[4, 5, 2] = 1
		# 	...and the black pieces on the sides of the eye
		one_hot_liberties[3, 3, 2] = 1
		one_hot_liberties[5, 3, 2] = 1

		# the black piece at (4,2) has 4 liberties
		one_hot_liberties[4, 2, 3] = 1

		for i in range(8):
			self.assertTrue(
				np.all(feature[:, :, i] == one_hot_liberties[:, :, i]),
				"bad expectation: stones with %d liberties" % (i + 1))

	def test_get_capture_size(self):
		# TODO - at the moment there is no imminent capture
		gs = simple_board()
		pp = Preprocess(["capture_size"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		one_hot_capture = np.zeros((gs.size, gs.size, 8))
		# there is no capture available; all legal moves are zero-capture
		for (x, y) in gs.get_legal_moves():
			one_hot_capture[x, y, 0] = 1

		for i in range(8):
			self.assertTrue(
				np.all(feature[:, :, i] == one_hot_capture[:, :, i]),
				"bad expectation: capturing %d stones" % i)

	def test_get_self_atari_size(self):
		# TODO - at the moment there is no imminent self-atari for white
		gs = simple_board()
		pp = Preprocess(["self_atari_size"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		self.assertTrue(np.all(feature == np.zeros((gs.size, gs.size, 8))))

	def test_get_liberties_after(self):
		gs = simple_board()
		pp = Preprocess(["liberties_after"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		one_hot_liberties = np.zeros((gs.size, gs.size, 8))

		# TODO (?) hand-code?
		for (x, y) in gs.get_legal_moves():
			copy = gs.copy()
			copy.do_move((x, y))
			libs = copy.liberty_counts[x, y]
			if libs < 7:
				one_hot_liberties[x, y, libs] = 1
			else:
				one_hot_liberties[x, y, 7] = 1

		for i in range(8):
			self.assertTrue(
				np.all(feature[:, :, i] == one_hot_liberties[:, :, i]),
				"bad expectation: stones with %d liberties after move" % (i + 1))

	def test_get_ladder_capture(self):
		pass

	def test_get_ladder_escape(self):
		pass

	def test_get_sensibleness(self):
		# TODO - there are no legal eyes at the moment

		gs = simple_board()
		pp = Preprocess(["sensibleness"])
		feature = pp.state_to_tensor(gs)[0, 0]  # 1D tensor; no need to transpose

		expectation = np.zeros((gs.size, gs.size))
		for (x, y) in gs.get_legal_moves():
			if not (gs.is_eye((x, y), go.WHITE)):
				expectation[x, y] = 1
		self.assertTrue(np.all(expectation == feature))

	def test_feature_concatenation(self):
		gs = simple_board()
		pp = Preprocess(["board", "sensibleness", "capture_size"])
		feature = pp.state_to_tensor(gs)[0].transpose((1, 2, 0))

		expectation = np.zeros((gs.size, gs.size, 3 + 1 + 8))

		# first three planes: board
		expectation[:, :, 0] = (gs.board == go.WHITE) * 1
		expectation[:, :, 1] = (gs.board == go.BLACK) * 1
		expectation[:, :, 2] = (gs.board == go.EMPTY) * 1

		# 4th plane: sensibleness (as in test_get_sensibleness)
		for (x, y) in gs.get_legal_moves():
			if not (gs.is_eye((x, y), go.WHITE)):
				expectation[x, y, 3] = 1

		# 5th through 12th plane: capture size (all zero-capture)
		for (x, y) in gs.get_legal_moves():
			expectation[x, y, 4] = 1

		self.assertTrue(np.all(expectation == feature))

if __name__ == '__main__':
	unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

import os
from six import iteritems
import logging

from werkzeug.wrappers import Request
from werkzeug.local import LocalManager
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.contrib.profiler import ProfilerMiddleware
from werkzeug.wsgi import SharedDataMiddleware

import frappe
import frappe.handler
import frappe.auth
import frappe.api
import frappe.async
import frappe.utils.response
import frappe.website.render
from frappe.utils import get_site_name
from frappe.middlewares import StaticDataMiddleware
from frappe.utils.error import make_error_snapshot
from frappe.core.doctype.communication.comment import update_comments_in_parent_after_request
from frappe import _

# imports - third-party imports
import pymysql
from pymysql.constants import ER

# imports - module imports

local_manager = LocalManager([frappe.local])

_site = None
_sites_path = os.environ.get("SITES_PATH", ".")

class RequestContext(object):

	def __init__(self, environ):
		self.request = Request(environ)

	def __enter__(self):
		init_request(self.request)

	def __exit__(self, type, value, traceback):
		frappe.destroy()

@Request.application
def application(request):
	response = None

	try:
		rollback = True

		init_request(request)

		if frappe.local.form_dict.cmd:
			response = frappe.handler.handle()

		elif frappe.request.path.startswith("/api/"):
			if frappe.local.form_dict.data is None:
					frappe.local.form_dict.data = request.get_data()
			response = frappe.api.handle()

		elif frappe.request.path.startswith('/backups'):
			response = frappe.utils.response.download_backup(request.path)

		elif frappe.request.path.startswith('/private/files/'):
			response = frappe.utils.response.download_private_file(request.path)

		elif frappe.local.request.method in ('GET', 'HEAD', 'POST'):
			response = frappe.website.render.render()

		else:
			raise NotFound

	except HTTPException as e:
		return e

	except frappe.SessionStopped as e:
		response = frappe.utils.response.handle_session_stopped()

	except Exception as e:
		response = handle_exception(e)

	else:
		rollback = after_request(rollback)

	finally:
		if frappe.local.request.method in ("POST", "PUT") and frappe.db and rollback:
			frappe.db.rollback()

		# set cookies
		if response and hasattr(frappe.local, 'cookie_manager'):
			frappe.local.cookie_manager.flush_cookies(response=response)

		frappe.destroy()

	return response

def init_request(request):
	frappe.local.request = request
	frappe.local.is_ajax = frappe.get_request_header("X-Requested-With")=="XMLHttpRequest"

	site = _site or request.headers.get('X-Frappe-Site-Name') or get_site_name(request.host)
	frappe.init(site=site, sites_path=_sites_path)

	if not (frappe.local.conf and frappe.local.conf.db_name):
		# site does not exist
		raise NotFound

	if frappe.local.conf.get('maintenance_mode'):
		raise frappe.SessionStopped

	make_form_dict(request)

	frappe.local.http_request = frappe.auth.HTTPRequest()

def make_form_dict(request):
	import json

	if 'application/json' in (request.content_type or '') and request.data:
		args = json.loads(request.data)
	else:
		args = request.form or request.args

	try:
		frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \
			for k, v in iteritems(args) })
	except IndexError:
		frappe.local.form_dict = frappe._dict(args)

	if "_" in frappe.local.form_dict:
		# _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict
		frappe.local.form_dict.pop("_")

def handle_exception(e):
	response = None
	http_status_code = getattr(e, "http_status_code", 500)
	return_as_message = False

	if frappe.local.is_ajax or 'application/json' in frappe.get_request_header('Accept'):
		# handle ajax responses first
		# if the request is ajax, send back the trace or error message
		response = frappe.utils.response.report_error(http_status_code)

	elif (http_status_code==500
		and isinstance(e, pymysql.InternalError)
		and e.args[0] in (ER.LOCK_WAIT_TIMEOUT, ER.LOCK_DEADLOCK)):
			http_status_code = 508

	elif http_status_code==401:
		frappe.respond_as_web_page(_("Session Expired"),
			_("Your session has expired, please login again to continue."),
			http_status_code=http_status_code,  indicator_color='red')
		return_as_message = True

	elif http_status_code==403:
		frappe.respond_as_web_page(_("Not Permitted"),
			_("You do not have enough permissions to complete the action"),
			http_status_code=http_status_code,  indicator_color='red')
		return_as_message = True

	elif http_status_code==404:
		frappe.respond_as_web_page(_("Not Found"),
			_("The resource you are looking for is not available"),
			http_status_code=http_status_code,  indicator_color='red')
		return_as_message = True

	else:
		traceback = "
"+frappe.get_traceback()+"
"
		if frappe.local.flags.disable_traceback:
			traceback = ""

		frappe.respond_as_web_page("Server Error",
			traceback, http_status_code=http_status_code,
			indicator_color='red', width=640)
		return_as_message = True

	if e.__class__ == frappe.AuthenticationError:
		if hasattr(frappe.local, "login_manager"):
			frappe.local.login_manager.clear_cookies()

	if http_status_code >= 500:
		frappe.logger().error('Request Error', exc_info=True)
		make_error_snapshot(e)

	if return_as_message:
		response = frappe.website.render.render("message",
			http_status_code=http_status_code)

	return response

def after_request(rollback):
	if (frappe.local.request.method in ("POST", "PUT") or frappe.local.flags.commit) and frappe.db:
		if frappe.db.transaction_writes:
			frappe.db.commit()
			rollback = False

	# update session
	if getattr(frappe.local, "session_obj", None):
		updated_in_db = frappe.local.session_obj.update()
		if updated_in_db:
			frappe.db.commit()
			rollback = False

	update_comments_in_parent_after_request()

	return rollback

application = local_manager.make_middleware(application)

def serve(port=8000, profile=False, site=None, sites_path='.'):
	global application, _site, _sites_path
	_site = site
	_sites_path = sites_path

	from werkzeug.serving import run_simple

	if profile:
		application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls'))

	if not os.environ.get('NO_STATICS'):
		application = SharedDataMiddleware(application, {
			'/assets': os.path.join(sites_path, 'assets'),
		})

		application = StaticDataMiddleware(application, {
			'/files': os.path.abspath(sites_path)
		})

	application.debug = True
	application.config = {
		'SERVER_NAME': 'localhost:8000'
	}

	in_test_env = os.environ.get('CI')
	if in_test_env:
		log = logging.getLogger('werkzeug')
		log.setLevel(logging.ERROR)

	run_simple('0.0.0.0', int(port), application,
		use_reloader=not in_test_env,
		use_debugger=not in_test_env,
		use_evalex=not in_test_env,
		threaded=True)
<commit_msg>Check if server is alive and let socket.io then request<commit_after># -*- coding: utf-8 -*-
# Copyright (c) 2015, Frappe Technologies Pvt. Ltd. and Contributors
# MIT License. See license.txt
from __future__ import unicode_literals

import os
from six import iteritems
import logging

from werkzeug.wrappers import Request
from werkzeug.local import LocalManager
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.contrib.profiler import ProfilerMiddleware
from werkzeug.wsgi import SharedDataMiddleware

import frappe
import frappe.handler
import frappe.auth
import frappe.api
import frappe.async
import frappe.utils.response
import frappe.website.render
from frappe.utils import get_site_name
from frappe.middlewares import StaticDataMiddleware
from frappe.utils.error import make_error_snapshot
from frappe.core.doctype.communication.comment import update_comments_in_parent_after_request
from frappe import _

# imports - third-party imports
import pymysql
from pymysql.constants import ER

# imports - module imports

local_manager = LocalManager([frappe.local])

_site = None
_sites_path = os.environ.get("SITES_PATH", ".")

class RequestContext(object):

	def __init__(self, environ):
		self.request = Request(environ)

	def __enter__(self):
		init_request(self.request)

	def __exit__(self, type, value, traceback):
		frappe.destroy()

@Request.application
def application(request):
	response = None

	try:
		rollback = True

		init_request(request)

		if frappe.local.form_dict.cmd:
			response = frappe.handler.handle()

		elif frappe.request.path.startswith("/api/"):
			if frappe.local.form_dict.data is None:
					frappe.local.form_dict.data = request.get_data()
			response = frappe.api.handle()

		elif frappe.request.path.startswith('/backups'):
			response = frappe.utils.response.download_backup(request.path)

		elif frappe.request.path.startswith('/private/files/'):
			response = frappe.utils.response.download_private_file(request.path)

		elif frappe.local.request.method in ('GET', 'HEAD', 'POST'):
			response = frappe.website.render.render()

		else:
			raise NotFound

	except HTTPException as e:
		return e

	except frappe.SessionStopped as e:
		response = frappe.utils.response.handle_session_stopped()

	except Exception as e:
		response = handle_exception(e)

	else:
		rollback = after_request(rollback)

	finally:
		if frappe.local.request.method in ("POST", "PUT") and frappe.db and rollback:
			frappe.db.rollback()

		# set cookies
		if response and hasattr(frappe.local, 'cookie_manager'):
			frappe.local.cookie_manager.flush_cookies(response=response)

		frappe.destroy()

	return response

def init_request(request):
	frappe.local.request = request
	frappe.local.is_ajax = frappe.get_request_header("X-Requested-With")=="XMLHttpRequest"

	site = _site or request.headers.get('X-Frappe-Site-Name') or get_site_name(request.host)
	frappe.init(site=site, sites_path=_sites_path)

	if not (frappe.local.conf and frappe.local.conf.db_name):
		# site does not exist
		raise NotFound

	if frappe.local.conf.get('maintenance_mode'):
		raise frappe.SessionStopped

	make_form_dict(request)

	frappe.local.http_request = frappe.auth.HTTPRequest()

def make_form_dict(request):
	import json

	if 'application/json' in (request.content_type or '') and request.data:
		args = json.loads(request.data)
	else:
		args = request.form or request.args

	try:
		frappe.local.form_dict = frappe._dict({ k:v[0] if isinstance(v, (list, tuple)) else v \
			for k, v in iteritems(args) })
	except IndexError:
		frappe.local.form_dict = frappe._dict(args)

	if "_" in frappe.local.form_dict:
		# _ is passed by $.ajax so that the request is not cached by the browser. So, remove _ from form_dict
		frappe.local.form_dict.pop("_")

def handle_exception(e):
	response = None
	http_status_code = getattr(e, "http_status_code", 500)
	return_as_message = False

	if frappe.get_request_header('Accept') and frappe.local.is_ajax or 'application/json' in frappe.get_request_header('Accept'):
		# handle ajax responses first
		# if the request is ajax, send back the trace or error message
		response = frappe.utils.response.report_error(http_status_code)

	elif (http_status_code==500
		and isinstance(e, pymysql.InternalError)
		and e.args[0] in (ER.LOCK_WAIT_TIMEOUT, ER.LOCK_DEADLOCK)):
			http_status_code = 508

	elif http_status_code==401:
		frappe.respond_as_web_page(_("Session Expired"),
			_("Your session has expired, please login again to continue."),
			http_status_code=http_status_code,  indicator_color='red')
		return_as_message = True

	elif http_status_code==403:
		frappe.respond_as_web_page(_("Not Permitted"),
			_("You do not have enough permissions to complete the action"),
			http_status_code=http_status_code,  indicator_color='red')
		return_as_message = True

	elif http_status_code==404:
		frappe.respond_as_web_page(_("Not Found"),
			_("The resource you are looking for is not available"),
			http_status_code=http_status_code,  indicator_color='red')
		return_as_message = True

	else:
		traceback = "
"+frappe.get_traceback()+"
"
		if frappe.local.flags.disable_traceback:
			traceback = ""

		frappe.respond_as_web_page("Server Error",
			traceback, http_status_code=http_status_code,
			indicator_color='red', width=640)
		return_as_message = True

	if e.__class__ == frappe.AuthenticationError:
		if hasattr(frappe.local, "login_manager"):
			frappe.local.login_manager.clear_cookies()

	if http_status_code >= 500:
		frappe.logger().error('Request Error', exc_info=True)
		make_error_snapshot(e)

	if return_as_message:
		response = frappe.website.render.render("message",
			http_status_code=http_status_code)

	return response

def after_request(rollback):
	if (frappe.local.request.method in ("POST", "PUT") or frappe.local.flags.commit) and frappe.db:
		if frappe.db.transaction_writes:
			frappe.db.commit()
			rollback = False

	# update session
	if getattr(frappe.local, "session_obj", None):
		updated_in_db = frappe.local.session_obj.update()
		if updated_in_db:
			frappe.db.commit()
			rollback = False

	update_comments_in_parent_after_request()

	return rollback

application = local_manager.make_middleware(application)

def serve(port=8000, profile=False, site=None, sites_path='.'):
	global application, _site, _sites_path
	_site = site
	_sites_path = sites_path

	from werkzeug.serving import run_simple

	if profile:
		application = ProfilerMiddleware(application, sort_by=('cumtime', 'calls'))

	if not os.environ.get('NO_STATICS'):
		application = SharedDataMiddleware(application, {
			'/assets': os.path.join(sites_path, 'assets'),
		})

		application = StaticDataMiddleware(application, {
			'/files': os.path.abspath(sites_path)
		})

	application.debug = True
	application.config = {
		'SERVER_NAME': 'localhost:8000'
	}

	in_test_env = os.environ.get('CI')
	if in_test_env:
		log = logging.getLogger('werkzeug')
		log.setLevel(logging.ERROR)

	run_simple('0.0.0.0', int(port), application,
		use_reloader=not in_test_env,
		use_debugger=not in_test_env,
		use_evalex=not in_test_env,
		threaded=True)
<|endoftext|>


<commit_before>#!/usr/bin/python

"""
cloudviz.py
This script exposes Amazon EC2 CloudWatch as a data source for the Google Visualization API

Requirements:
- AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, read in from settings.py
- boto, a Python interface for Amazon Web Services (http://code.google.com/p/boto/)
- gviz_api, a Python library for creating Google Visualization API data sources
  (http://code.google.com/p/google-visualization-python/)
- pytz, world timezone definitions for Python (http://pytz.sourceforge.net/)

Cloudviz project maintained here: http://github.com/mbabineau/cloudviz
--------
Copyright 2010 Bizo, Inc. (Mike Babineau <michael.babineau@gmail.com>)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import sys
import cgi
import operator
from datetime import datetime, timedelta
from django.utils import simplejson
from pytz import timezone
import pytz

# Google Visualization API
import gviz_api

import boto.ec2.cloudwatch
from boto.ec2.cloudwatch.metric import Metric

from settings import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, DEFAULTS, CW_MAX_DATA_POINTS, CW_MIN_PERIOD

def get_cloudwatch_data(cloudviz_query, request_id, aws_access_key_id=None, aws_secret_access_key=None):
    """
    Query CloudWatch and return the results in a Google Visualizations API-friendly format

    Arguments:
    `cloudviz_query` -- (dict) parameters and values to be passed to CloudWatch (see README for more information)
    `request_id` -- (int) Google Visualizations request ID passed as part of the "tqx" parameter
    """
    # Initialize data description, columns to be returned, and result set
    description = { "Timestamp": ("datetime", "Timestamp")}
    columns = ["Timestamp"]
    rs = []
    current_timezone = timezone('UTC')
    utc = pytz.utc

    # Build option list
    opts = ['unit','metric','namespace','statistics','period', 'dimensions', 'prefix',
            'start_time', 'end_time', 'calc_rate', 'region', 'range', 'timezone']

    # Set default parameter values from config.py
    qa = DEFAULTS.copy()

    # Set passed args
    for opt in opts:
        if opt in cloudviz_query: qa[opt] = cloudviz_query[opt]

    # Convert timestamps to datetimes if necessary
    for time in ['start_time','end_time']:
        if time in qa:
            if type(qa[time]) == str or type(qa[time]) == unicode:
                qa[time] = datetime.strptime(qa[time].split(".")[0], '%Y-%m-%dT%H:%M:%S')

    # If both start_time and end_time are specified, do nothing.
    if 'start_time' in qa and 'end_time' in qa:
        pass
    # If only one of the times is specified, fill in the other based on range
    elif 'start_time' in qa and 'range' in qa:
        qa['end_time'] = qa['start_time'] + timedelta(hours=qa['range'])
    elif 'range' in qa and 'end_time' in qa:
        qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range'])
    # If neither is specified, use range leading up to current time
    else:
        qa['end_time'] = datetime.now()
        qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range'])

    if 'timezone' in qa:
        current_timezone = timezone(qa['timezone'])

    # Parse, build, and run each CloudWatch query
    cloudwatch_opts = ['unit', 'metric', 'namespace', 'statistics', 'period', 'dimensions', 'prefix', 'calc_rate', 'region']
    for cloudwatch_query in cloudviz_query['cloudwatch_queries']:
        args = qa.copy()
        # Override top-level vars
        for opt in cloudwatch_opts:
            if opt in cloudwatch_query: args[opt] = cloudwatch_query[opt]

        # Calculate time range for period determination/sanity-check
        delta = args['end_time'] - args['start_time']
        delta_seconds = ( delta.days * 24 * 60 * 60 ) + delta.seconds + 1 #round microseconds up

        # Determine min period as the smallest multiple of 60 that won't result in too many data points
        min_period = 60 * int(delta_seconds / CW_MAX_DATA_POINTS / 60)
        if ((delta_seconds / CW_MAX_DATA_POINTS) % 60) > 0:
            min_period += 60

        if 'period' in qa:
            if args['period'] < min_period:
                args['period'] = min_period
        else:
            args['period'] = min_period

        # Make sure period isn't smaller than CloudWatch allows
        if args['period'] < CW_MIN_PERIOD:
            args['period'] = CW_MIN_PERIOD

        # Defaulting AWS region to us-east-1
        if not 'region' in args:
            args['region'] = 'us-east-1'

        # Use AWS keys if provided, otherwise just let the boto look it up
        if aws_access_key_id and aws_secret_access_key:
            c = boto.ec2.cloudwatch.connect_to_region(  args['region'], aws_access_key_id=AWS_ACCESS_KEY_ID,
                                                        aws_secret_access_key=AWS_SECRET_ACCESS_KEY, is_secure=False)
        else:
            boto.ec2.cloudwatch.connect_to_region(args['region'], is_secure=False)

        # Pull data from EC2
        results = c.get_metric_statistics(  args['period'], args['start_time'], args['end_time'],
                                            args['metric'], args['namespace'], args['statistics'],
                                            args['dimensions'], args['unit'])
        # Format/transform results
        for d in results:
            # Convert timestamps to datetime objects
            d.update({u'Timestamp': d[u'Timestamp']})
            utc_dt = utc.localize(d[u'Timestamp'])
            loc_dt = utc_dt.astimezone(current_timezone)
            d['Timestamp'] = loc_dt
            # If desired, convert Sum to a per-second Rate
            if args['calc_rate'] == True and 'Sum' in args['statistics']: d.update({u'Rate': d[u'Sum']/args['period']})
            # Change key names
            keys = d.keys()
            keys.remove('Timestamp')
            for k in keys:
                new_k = args['prefix']+k
                d[new_k] = d[k]
                del d[k]

        rs.extend(results)

        # Build data description and columns to be return
        description[args['prefix']+'Samples'] = ('number', args['prefix']+'Samples')
        description[args['prefix']+'Unit'] = ('string', args['unit'])
        for stat in args['statistics']:
            # If Rate is desired, update label accordingly
            if stat == 'Sum' and args['calc_rate'] == True:
                stat = 'Rate'
            description[args['prefix']+stat] = ('number', args['prefix']+stat)
            columns.append(args['prefix']+stat)

    # Sort data and present
    data = sorted(rs, key=operator.itemgetter(u'Timestamp'))
    data_table = gviz_api.DataTable(description)
    data_table.LoadData(data)

    results = data_table.ToJSonResponse(columns_order=columns, order_by="Timestamp", req_id=request_id)
    return results

def main():
    # Parse the query string
    fs = cgi.FieldStorage()
    cloudviz_query = simplejson.loads(fs.getvalue('qs'))

    # Convert tqx to dict; tqx is a set of colon-delimited key/value pairs separated by semicolons
    tqx = {}
    for s in fs.getvalue('tqx').split(';'):
        key = s.split(':')[0]
        value = s.split(':')[1]
        tqx.update({key:value})

    # Set reqId so we know who to send data back to
    request_id = tqx['reqId']

    results = get_cloudwatch_data(cloudviz_query, request_id, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
    print "Content-type: text/plain"
    print
    print results

if __name__ == "__main__":
    status = main()
    sys.exit(status)
<commit_msg>Fix using boto to fetch AWS credentials<commit_after>#!/usr/bin/python

"""
cloudviz.py
This script exposes Amazon EC2 CloudWatch as a data source for the Google Visualization API

Requirements:
- AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, read in from settings.py
- boto, a Python interface for Amazon Web Services (http://code.google.com/p/boto/)
- gviz_api, a Python library for creating Google Visualization API data sources
  (http://code.google.com/p/google-visualization-python/)
- pytz, world timezone definitions for Python (http://pytz.sourceforge.net/)

Cloudviz project maintained here: http://github.com/mbabineau/cloudviz
--------
Copyright 2010 Bizo, Inc. (Mike Babineau <michael.babineau@gmail.com>)

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import sys
import cgi
import operator
from datetime import datetime, timedelta
from django.utils import simplejson
from pytz import timezone
import pytz

# Google Visualization API
import gviz_api

import boto.ec2.cloudwatch
from boto.ec2.cloudwatch.metric import Metric

from settings import AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, DEFAULTS, CW_MAX_DATA_POINTS, CW_MIN_PERIOD

def get_cloudwatch_data(cloudviz_query, request_id, aws_access_key_id=None, aws_secret_access_key=None):
    """
    Query CloudWatch and return the results in a Google Visualizations API-friendly format

    Arguments:
    `cloudviz_query` -- (dict) parameters and values to be passed to CloudWatch (see README for more information)
    `request_id` -- (int) Google Visualizations request ID passed as part of the "tqx" parameter
    """
    # Initialize data description, columns to be returned, and result set
    description = { "Timestamp": ("datetime", "Timestamp")}
    columns = ["Timestamp"]
    rs = []
    current_timezone = timezone('UTC')
    utc = pytz.utc

    # Build option list
    opts = ['unit','metric','namespace','statistics','period', 'dimensions', 'prefix',
            'start_time', 'end_time', 'calc_rate', 'region', 'range', 'timezone']

    # Set default parameter values from config.py
    qa = DEFAULTS.copy()

    # Set passed args
    for opt in opts:
        if opt in cloudviz_query: qa[opt] = cloudviz_query[opt]

    # Convert timestamps to datetimes if necessary
    for time in ['start_time','end_time']:
        if time in qa:
            if type(qa[time]) == str or type(qa[time]) == unicode:
                qa[time] = datetime.strptime(qa[time].split(".")[0], '%Y-%m-%dT%H:%M:%S')

    # If both start_time and end_time are specified, do nothing.
    if 'start_time' in qa and 'end_time' in qa:
        pass
    # If only one of the times is specified, fill in the other based on range
    elif 'start_time' in qa and 'range' in qa:
        qa['end_time'] = qa['start_time'] + timedelta(hours=qa['range'])
    elif 'range' in qa and 'end_time' in qa:
        qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range'])
    # If neither is specified, use range leading up to current time
    else:
        qa['end_time'] = datetime.now()
        qa['start_time'] = qa['end_time'] - timedelta(hours=qa['range'])

    if 'timezone' in qa:
        current_timezone = timezone(qa['timezone'])

    # Parse, build, and run each CloudWatch query
    cloudwatch_opts = ['unit', 'metric', 'namespace', 'statistics', 'period', 'dimensions', 'prefix', 'calc_rate', 'region']
    for cloudwatch_query in cloudviz_query['cloudwatch_queries']:
        args = qa.copy()
        # Override top-level vars
        for opt in cloudwatch_opts:
            if opt in cloudwatch_query: args[opt] = cloudwatch_query[opt]

        # Calculate time range for period determination/sanity-check
        delta = args['end_time'] - args['start_time']
        delta_seconds = ( delta.days * 24 * 60 * 60 ) + delta.seconds + 1 #round microseconds up

        # Determine min period as the smallest multiple of 60 that won't result in too many data points
        min_period = 60 * int(delta_seconds / CW_MAX_DATA_POINTS / 60)
        if ((delta_seconds / CW_MAX_DATA_POINTS) % 60) > 0:
            min_period += 60

        if 'period' in qa:
            if args['period'] < min_period:
                args['period'] = min_period
        else:
            args['period'] = min_period

        # Make sure period isn't smaller than CloudWatch allows
        if args['period'] < CW_MIN_PERIOD:
            args['period'] = CW_MIN_PERIOD

        # Defaulting AWS region to us-east-1
        if not 'region' in args:
            args['region'] = 'us-east-1'

        # Use AWS keys if provided, otherwise just let the boto look it up
        if aws_access_key_id and aws_secret_access_key:
            c = boto.ec2.cloudwatch.connect_to_region(  args['region'], aws_access_key_id=AWS_ACCESS_KEY_ID,
                                                        aws_secret_access_key=AWS_SECRET_ACCESS_KEY, is_secure=False)
        else:
            c = boto.ec2.cloudwatch.connect_to_region(args['region'], is_secure=False)

        # Pull data from EC2
        results = c.get_metric_statistics(  args['period'], args['start_time'], args['end_time'],
                                            args['metric'], args['namespace'], args['statistics'],
                                            args['dimensions'], args['unit'])
        # Format/transform results
        for d in results:
            # Convert timestamps to datetime objects
            d.update({u'Timestamp': d[u'Timestamp']})
            utc_dt = utc.localize(d[u'Timestamp'])
            loc_dt = utc_dt.astimezone(current_timezone)
            d['Timestamp'] = loc_dt
            # If desired, convert Sum to a per-second Rate
            if args['calc_rate'] == True and 'Sum' in args['statistics']: d.update({u'Rate': d[u'Sum']/args['period']})
            # Change key names
            keys = d.keys()
            keys.remove('Timestamp')
            for k in keys:
                new_k = args['prefix']+k
                d[new_k] = d[k]
                del d[k]

        rs.extend(results)

        # Build data description and columns to be return
        description[args['prefix']+'Samples'] = ('number', args['prefix']+'Samples')
        description[args['prefix']+'Unit'] = ('string', args['unit'])
        for stat in args['statistics']:
            # If Rate is desired, update label accordingly
            if stat == 'Sum' and args['calc_rate'] == True:
                stat = 'Rate'
            description[args['prefix']+stat] = ('number', args['prefix']+stat)
            columns.append(args['prefix']+stat)

    # Sort data and present
    data = sorted(rs, key=operator.itemgetter(u'Timestamp'))
    data_table = gviz_api.DataTable(description)
    data_table.LoadData(data)

    results = data_table.ToJSonResponse(columns_order=columns, order_by="Timestamp", req_id=request_id)
    return results

def main():
    # Parse the query string
    fs = cgi.FieldStorage()
    cloudviz_query = simplejson.loads(fs.getvalue('qs'))

    # Convert tqx to dict; tqx is a set of colon-delimited key/value pairs separated by semicolons
    tqx = {}
    for s in fs.getvalue('tqx').split(';'):
        key = s.split(':')[0]
        value = s.split(':')[1]
        tqx.update({key:value})

    # Set reqId so we know who to send data back to
    request_id = tqx['reqId']

    results = get_cloudwatch_data(cloudviz_query, request_id, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
    print "Content-type: text/plain"
    print
    print results

if __name__ == "__main__":
    status = main()
    sys.exit(status)
<|endoftext|>


<commit_before>"""
Symfony 2 deployment

"""
from __future__ import with_statement

# Add install directory to the beginning of the include path
import sys
import os

if os.path.exists("app/config/deployment/custom.py"):
    sys.path[:0] = ["app/config/deployment"]

sys.path[:0] = ["vendor/hgg/sfdeploy/bin"]

from fabric.api import *
from fabric.contrib.files import *
from fabric.contrib.console import confirm
from fabric.colors import red, green
from datetime import date, timedelta
import operator
import time
import config
import git
import pear
import shell
import tools

try:
    import custom
except ImportError:
    print(red('Cannot import custom. Any custom fabric task will not be available.'))

def load_config():
    """
    Load the yaml configuration file into the env dictionary.
    This depends on the deployment_target being set and should typically get
    called from either dev() or prod().
    """
    env.update(config.load_yaml_config('app/config/deployment/config.yml',
               env.deployment_target))

@task
def dev():
    """
    Set the target to be the development environment and load the configuration
    for that environment.
    """
    env.deployment_target = 'dev'
    load_config()

@task
def prod():
    """
    Set the target to be the production environment and load the configuration
    for that environment.
    """
    env.deployment_target = 'prod'
    load_config()

@task
def deploy():
    """
    Deploy to project to target server(s)
    """
    if git.is_git_dirty():
        if not confirm(red('Your working copy is dirty! You have not committed your code changes. Are you sure you want to continue?', bold=True)):
            return

        if env.deployment_target == 'prod':
            print(red('Deploying a dirty copy to prod is very naughty and not permitted', bold=True))
            exit(1)

        if not confirm(red('You are about to deploy a DIRTY copy to target "%s". Continue?' %
                       (env.deployment_target), bold=True)):
            return
    else:
        if not confirm(red('You are about to deploy the commit "%s" copy to target "%s". Continue?' %
                       (git.git_sha1_commit(), env.deployment_target),
                       bold=True)):
            return

    deploy_rev = git.git_sha1_commit()
    set_source_dir(deploy_rev)
    set_tmp_dir(deploy_rev)

    print(green('Deploying revision %s to "%s"' %
          (deploy_rev, env.deployment_target), bold=True))
    make_folders()
    mkdir(env.source_dir, 755)
    upload_source(deploy_rev, env.directories['packages']['dir'], env.source_dir)
    mkdir('%s/app/cache' % (env.source_dir), 777)
    mkdir('%s/app/logs' % (env.source_dir), 777)
    sf_permissions()
    post_upload_hook()
    stop()
    link_folders()
    start()
    cleanup()
    print(green('Successfully deployed revision %s to %s' %
          (deploy_rev, env.deployment_target), bold=True))

@task
def installed_rev():
    """
    Get the installed revision on the target server
    """
    print(green('Installed revision on %s: %s' %
          (env.deployment_target, get_installed_revision())))

def get_installed_revision():
    """
    Not sure if this will work too well anymore until the actual git revision
    is used.
    """
    parts = get_current_physical_dir().split('/')

    return parts.pop()

def sf_permissions():
    print('You might want to do something here')
    #sudo('cd %s && ./symfony project:permissions' % (env.source_dir))

def make_folders():
    """
    Create the folder structure on the deployment target
    """
    print(green('Creating remote folders'))
    sorted = []

    for dir, values in env.directories.iteritems():
        sorted.append(values)

    sorted.sort(key=operator.itemgetter('order'))

    for values in sorted:
        mkdir(values['dir'], values['perms'])

def mkdir(path, mod):
    if not exists('%s' % (path)):
        sudo('mkdir %s' % (path))
        sudo('chmod %s %s' % (mod, path))

def set_source_dir(deploy_rev = ''):
    if deploy_rev != '':
        env.source_dir = "%s/%s" % (env.directories['sources']['dir'], deploy_rev)
    else:
        env.source_dir = get_current_physical_dir()

def set_tmp_dir(deploy_rev):
    env.tmp_dir = '%s/%s' % (tempfile.gettempdir(), deploy_rev)

def get_current_physical_dir():
    if exists(env.symlinks['current']['path']):
        return sudo('cd %s && pwd -P' % (env.symlinks['current']['path']))
    else:
        print(red('Directory %s does not exist.' % env.symlinks['current']['path'], bold=True))
        raise Exception('Directory %s does not exist' % (env.symlinks['current']['path']))

def upload_source(deploy_revision, package_dir, source_dir):
    """
    Upload the project source code to the target server

    Suggested folder structure on the deployment target:

      project_dir
        |-current (symlink to source_dir/<deploy_revision>)
        |-previous (symlink to previous source_dir/<deploy_revision>)
        |-package_dir
        |   |-<deploy_revision>.tar.gz
        |
        |-source_dir
        |   |-<deploy_revision>
        |   |    |-the project source

    Steps taken by this task:

      * Create an archive of the source code
      * Upload it to the package folder
      * Unpack it in a versioned release directory
      * Remove the archive on the local system
    """
    print(green('Creating source archive'))
    archive_name = '%s.tar.gz' % (deploy_revision)
    shell.archive_all(os.getcwd(), archive_name, env.ignore_on_deploy)
    print(green('Uploading archive to target'))
    put('%s' % (archive_name), '%s/'  % (package_dir), True)
    sudo('cd %s && tar zxf %s/%s' % (source_dir, package_dir, archive_name))
    local('rm %s' % archive_name)

def link_folders():
    """
    Link package folder to current and the previous folder to the previously
    installed package folder.
    """
    print(green('Linking folders', bold=True))
    if exists(env.symlinks['current']['path']):
        sudo('rm -rf %s' % (env.symlinks['current']['path']))

    sudo('ln -s %s %s' % (env.source_dir, env.symlinks['current']['path']))

def cleanup():
    sudo('rm -rf %s/app/cache/*' % (env.source_dir))
    sudo('rm -rf %s/app/logs/*' % (env.source_dir))

@task
def start():
    """
    Start all cron jobs
    """
    print(green('Starting cron jobs', bold=True))
    try:
        physical_dir = get_current_physical_dir()

        if env.symlinks['current']['path']!= physical_dir:
            cron = load_cron_config()
            for job in cron['cron']['jobs']:
                install_sf_cron_job(job, cron['cron']['runhour'][env.deployment_target], physical_dir)
        else:
            print(red('No cron jobs to start'))
    except Exception as inst:
        print(red('Cron jobs not started. Exception: %s' % inst, bold=True))

@task
def stop():
    """
    Stop all cron jobs
    """
    print(green('Stopping cron jobs', bold=True))
    try:
        physical_dir = get_current_physical_dir()

        if env.symlinks['current']['path']!= physical_dir:
            cron = load_cron_config()
            with settings(warn_only=True):
                sudo('rm %s/%s*' % (cron['cron']['cron_dir'], cron['cron']['cron_job_prefix']))
        else:
            print(red('No cron jobs to stop', bold=True))
    except Exception as inst:
        print(red('Cron jobs not stopped (There might not be any installed). Exception: %s' % inst, bold=True))

def load_cron_config():
    cron = config.load_yaml('config/deployment/cron.yml')

    return cron

def post_upload_hook():
    if ('custom' in sys.modules):
      try:
          custom.post_upload_hook()
      except AttributeError:
          print('No post_upload_hook defined')

def install_sf_cron_job(job, hour, install_dir):
    """
    Install a symfony cron job
    """
    if env.deployment_target in job['targets']:
        print(green('Installing cron job %s:%s' %
              (job['namespace'], job['name']), bold=True))
        args_opts = ' '

        for index, object in enumerate(job['options']):
            if '##cron-hour##' in object:
                job['options'][index] = object.replace('##cron-hour##', str(hour))

        if not '--install' in job['options']:
            job['options'].insert(0, '--install')

        ao = job['arguments']
        ao.extend(job['options'])
        args_opts = args_opts.join(ao)

        cmd = ("%s/symfony %s:%s %s" %
               (install_dir, job['namespace'], job['name'], args_opts))
        sudo(cmd)
    else:
        print(green('Skipped %s:%s' %
              (job['namespace'], job['name']), bold=True))
<commit_msg>Fix path in cron config file load<commit_after>"""
Symfony 2 deployment

"""
from __future__ import with_statement

# Add install directory to the beginning of the include path
import sys
import os

if os.path.exists("app/config/deployment/custom.py"):
    sys.path[:0] = ["app/config/deployment"]

sys.path[:0] = ["vendor/hgg/sfdeploy/bin"]

from fabric.api import *
from fabric.contrib.files import *
from fabric.contrib.console import confirm
from fabric.colors import red, green
from datetime import date, timedelta
import operator
import time
import config
import git
import pear
import shell
import tools

try:
    import custom
except ImportError:
    print(red('Cannot import custom. Any custom fabric task will not be available.'))

def load_config():
    """
    Load the yaml configuration file into the env dictionary.
    This depends on the deployment_target being set and should typically get
    called from either dev() or prod().
    """
    env.update(config.load_yaml_config('app/config/deployment/config.yml',
               env.deployment_target))

@task
def dev():
    """
    Set the target to be the development environment and load the configuration
    for that environment.
    """
    env.deployment_target = 'dev'
    load_config()

@task
def prod():
    """
    Set the target to be the production environment and load the configuration
    for that environment.
    """
    env.deployment_target = 'prod'
    load_config()

@task
def deploy():
    """
    Deploy to project to target server(s)
    """
    if git.is_git_dirty():
        if not confirm(red('Your working copy is dirty! You have not committed your code changes. Are you sure you want to continue?', bold=True)):
            return

        if env.deployment_target == 'prod':
            print(red('Deploying a dirty copy to prod is very naughty and not permitted', bold=True))
            exit(1)

        if not confirm(red('You are about to deploy a DIRTY copy to target "%s". Continue?' %
                       (env.deployment_target), bold=True)):
            return
    else:
        if not confirm(red('You are about to deploy the commit "%s" copy to target "%s". Continue?' %
                       (git.git_sha1_commit(), env.deployment_target),
                       bold=True)):
            return

    deploy_rev = git.git_sha1_commit()
    set_source_dir(deploy_rev)
    set_tmp_dir(deploy_rev)

    print(green('Deploying revision %s to "%s"' %
          (deploy_rev, env.deployment_target), bold=True))
    make_folders()
    mkdir(env.source_dir, 755)
    upload_source(deploy_rev, env.directories['packages']['dir'], env.source_dir)
    mkdir('%s/app/cache' % (env.source_dir), 777)
    mkdir('%s/app/logs' % (env.source_dir), 777)
    sf_permissions()
    post_upload_hook()
    stop()
    link_folders()
    start()
    cleanup()
    print(green('Successfully deployed revision %s to %s' %
          (deploy_rev, env.deployment_target), bold=True))

@task
def installed_rev():
    """
    Get the installed revision on the target server
    """
    print(green('Installed revision on %s: %s' %
          (env.deployment_target, get_installed_revision())))

def get_installed_revision():
    """
    Not sure if this will work too well anymore until the actual git revision
    is used.
    """
    parts = get_current_physical_dir().split('/')

    return parts.pop()

def sf_permissions():
    print('You might want to do something here')
    #sudo('cd %s && ./symfony project:permissions' % (env.source_dir))

def make_folders():
    """
    Create the folder structure on the deployment target
    """
    print(green('Creating remote folders'))
    sorted = []

    for dir, values in env.directories.iteritems():
        sorted.append(values)

    sorted.sort(key=operator.itemgetter('order'))

    for values in sorted:
        mkdir(values['dir'], values['perms'])

def mkdir(path, mod):
    if not exists('%s' % (path)):
        sudo('mkdir %s' % (path))
        sudo('chmod %s %s' % (mod, path))

def set_source_dir(deploy_rev = ''):
    if deploy_rev != '':
        env.source_dir = "%s/%s" % (env.directories['sources']['dir'], deploy_rev)
    else:
        env.source_dir = get_current_physical_dir()

def set_tmp_dir(deploy_rev):
    env.tmp_dir = '%s/%s' % (tempfile.gettempdir(), deploy_rev)

def get_current_physical_dir():
    if exists(env.symlinks['current']['path']):
        return sudo('cd %s && pwd -P' % (env.symlinks['current']['path']))
    else:
        print(red('Directory %s does not exist.' % env.symlinks['current']['path'], bold=True))
        raise Exception('Directory %s does not exist' % (env.symlinks['current']['path']))

def upload_source(deploy_revision, package_dir, source_dir):
    """
    Upload the project source code to the target server

    Suggested folder structure on the deployment target:

      project_dir
        |-current (symlink to source_dir/<deploy_revision>)
        |-previous (symlink to previous source_dir/<deploy_revision>)
        |-package_dir
        |   |-<deploy_revision>.tar.gz
        |
        |-source_dir
        |   |-<deploy_revision>
        |   |    |-the project source

    Steps taken by this task:

      * Create an archive of the source code
      * Upload it to the package folder
      * Unpack it in a versioned release directory
      * Remove the archive on the local system
    """
    print(green('Creating source archive'))
    archive_name = '%s.tar.gz' % (deploy_revision)
    shell.archive_all(os.getcwd(), archive_name, env.ignore_on_deploy)
    print(green('Uploading archive to target'))
    put('%s' % (archive_name), '%s/'  % (package_dir), True)
    sudo('cd %s && tar zxf %s/%s' % (source_dir, package_dir, archive_name))
    local('rm %s' % archive_name)

def link_folders():
    """
    Link package folder to current and the previous folder to the previously
    installed package folder.
    """
    print(green('Linking folders', bold=True))
    if exists(env.symlinks['current']['path']):
        sudo('rm -rf %s' % (env.symlinks['current']['path']))

    sudo('ln -s %s %s' % (env.source_dir, env.symlinks['current']['path']))

def cleanup():
    sudo('rm -rf %s/app/cache/*' % (env.source_dir))
    sudo('rm -rf %s/app/logs/*' % (env.source_dir))

@task
def start():
    """
    Start all cron jobs
    """
    print(green('Starting cron jobs', bold=True))
    try:
        physical_dir = get_current_physical_dir()

        if env.symlinks['current']['path']!= physical_dir:
            cron = load_cron_config()
            for job in cron['cron']['jobs']:
                install_sf_cron_job(job, cron['cron']['runhour'][env.deployment_target], physical_dir)
        else:
            print(red('No cron jobs to start'))
    except Exception as inst:
        print(red('Cron jobs not started. Exception: %s' % inst, bold=True))

@task
def stop():
    """
    Stop all cron jobs
    """
    print(green('Stopping cron jobs', bold=True))
    try:
        physical_dir = get_current_physical_dir()

        if env.symlinks['current']['path']!= physical_dir:
            cron = load_cron_config()
            with settings(warn_only=True):
                sudo('rm %s/%s*' % (cron['cron']['cron_dir'], cron['cron']['cron_job_prefix']))
        else:
            print(red('No cron jobs to stop', bold=True))
    except Exception as inst:
        print(red('Cron jobs not stopped (There might not be any installed). Exception: %s' % inst, bold=True))

def load_cron_config():
    cron = config.load_yaml('app/config/deployment/cron.yml')

    return cron

def post_upload_hook():
    if ('custom' in sys.modules):
      try:
          custom.post_upload_hook()
      except AttributeError:
          print('No post_upload_hook defined')

def install_sf_cron_job(job, hour, install_dir):
    """
    Install a symfony cron job
    """
    if env.deployment_target in job['targets']:
        print(green('Installing cron job %s:%s' %
              (job['namespace'], job['name']), bold=True))
        args_opts = ' '

        for index, object in enumerate(job['options']):
            if '##cron-hour##' in object:
                job['options'][index] = object.replace('##cron-hour##', str(hour))

        if not '--install' in job['options']:
            job['options'].insert(0, '--install')

        ao = job['arguments']
        ao.extend(job['options'])
        args_opts = args_opts.join(ao)

        cmd = ("%s/symfony %s:%s %s" %
               (install_dir, job['namespace'], job['name'], args_opts))
        sudo(cmd)
    else:
        print(green('Skipped %s:%s' %
              (job['namespace'], job['name']), bold=True))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# import math
import glob
import multiprocessing
from pathlib import Path

# import matplotlib.patches as mpatches
# import numpy as np
import pandas as pd
from astropy import constants as const
from astropy import units as u

# from collections import namedtuple
from functools import lru_cache

import artistools as at

columns = (
    'number',
    'where',
    'type_id',
    'posx', 'posy', 'posz',
    'dirx', 'diry', 'dirz',
    'last_cross',
    'tdecay',
    'e_cmf',
    'e_rf',
    'nu_cmf',
    'nu_rf',
    'escape_type_id',
    'escape_time',
    'scat_count',
    'next_trans',
    'interactions',
    'last_event',
    'emissiontype',
    'trueemissiontype',
    'em_posx', 'em_posy', 'em_posz',
    'absorption_type',
    'absorption_freq',
    'nscatterings',
    'em_time',
    'absorptiondirx',
    'absorptiondiry',
    'absorptiondirz', 'stokes1', 'stokes2', 'stokes3', 'pol_dirx', 'pol_diry',
    'pol_dirz',
    'originated_from_positron',
    'true_emission_velocity',
    'trueem_time',
)

types = {
    10: 'TYPE_GAMMA',
    11: 'TYPE_RPKT',
    20: 'TYPE_NTLEPTON',
    32: 'TYPE_ESCAPE',
}

type_ids = dict((v, k) for k, v in types.items())

def add_derived_columns(dfpackets, modelpath, colnames, allnonemptymgilist=None):
    if dfpackets.empty:
        return dfpackets

    colnames = at.makelist(colnames)

    def em_modelgridindex(packet):
        return at.get_mgi_of_velocity_kms(modelpath, packet.emission_velocity * u.cm.to('km'),
                                          mgilist=allnonemptymgilist)

    def emtrue_modelgridindex(packet):
        return at.get_mgi_of_velocity_kms(modelpath, packet.true_emission_velocity * u.cm.to('km'), mgilist=allnonemptymgilist)

    def em_timestep(packet):
        return at.get_timestep_of_timedays(modelpath, packet.em_time * u.s.to('day'))

    if 'emission_velocity' in colnames:
        dfpackets.eval(
            "emission_velocity = sqrt(em_posx ** 2 + em_posy ** 2 + em_posz ** 2) / em_time",
            inplace=True)

    if 'em_modelgridindex' in colnames:
        if 'emission_velocity' not in dfpackets.columns:
            dfpackets = add_derived_columns(dfpackets, modelpath, ['emission_velocity'],
                                            allnonemptymgilist=allnonemptymgilist)
        dfpackets['em_modelgridindex'] = dfpackets.apply(em_modelgridindex, axis=1)

    if 'emtrue_modelgridindex' in colnames:
        dfpackets['emtrue_modelgridindex'] = dfpackets.apply(emtrue_modelgridindex, axis=1)

    if 'em_timestep' in colnames:
        dfpackets['em_timestep'] = dfpackets.apply(em_timestep, axis=1)

    return dfpackets

@at.diskcache(savegzipped=True)
def readfile(packetsfile, type=None, escape_type=None):
    """Read a packet file into a pandas DataFrame."""
    filesize = Path(packetsfile).stat().st_size / 1024 / 1024

    print(f'Reading {packetsfile} ({filesize:.1f} MiB)', end='')
    inputcolumncount = len(pd.read_csv(packetsfile, nrows=1, delim_whitespace=True, header=None).columns)
    if inputcolumncount < 3:
        print("\nWARNING: packets file has no columns!")
        print(open(packetsfile, "r").readlines())

    # the packets file may have a truncated set of columns, but we assume that they
    # are only truncated, i.e. the columns with the same index have the same meaning
    usecols_nodata = [n for n in columns if columns.index(n) >= inputcolumncount]
    # usecols_actual = [n for n in columns if columns.index(n) < inputcolumncount]

    dfpackets = pd.read_csv(
        packetsfile, delim_whitespace=True,
        names=columns[:inputcolumncount], header=None)

    print(f' ({len(dfpackets):.1e} packets', end='')

    if escape_type is not None and escape_type != '' and escape_type != 'ALL':
        assert type is None or type == 'TYPE_ESCAPE'
        dfpackets.query(f'type_id == {type_ids["TYPE_ESCAPE"]} and escape_type_id == {type_ids[escape_type]}',
                        inplace=True)
        print(f', {len(dfpackets)} escaped as {escape_type})')
    elif type is not None and type != 'ALL' and type != '':
        dfpackets.query(f'type_id == {type_ids[type]}', inplace=True)
        print(f', {len(dfpackets)} with type {type})')
    else:
        print(')')

    # dfpackets['type'] = dfpackets['type_id'].map(lambda x: types.get(x, x))
    # dfpackets['escape_type'] = dfpackets['escape_type_id'].map(lambda x: types.get(x, x))

    if usecols_nodata:
        print(f'WARNING: no data in packets file for columns: {usecols_nodata}')
        for col in usecols_nodata:
            dfpackets[col] = float('NaN')

    # # neglect light travel time correction
    # dfpackets.eval("t_arrive_d = escape_time * @u.s.to('day')", inplace=True)

    dfpackets.eval(
        "t_arrive_d = (escape_time - "
        "(posx * dirx + posy * diry + posz * dirz) / @const.c.to('cm/s').value) * @u.s.to('day')", inplace=True)

    return dfpackets

@lru_cache(maxsize=16)
def get_packetsfilepaths(modelpath, maxpacketfiles=None):
    packetsfiles = sorted(
        glob.glob(str(Path(modelpath, 'packets00_*.out*'))) +
        glob.glob(str(Path(modelpath, 'packets', 'packets00_*.out*'))))
    if maxpacketfiles is not None and maxpacketfiles > 0 and len(packetsfiles) > maxpacketfiles:
        print(f'Using only the first {maxpacketfiles} of {len(packetsfiles)} packets files')
        packetsfiles = packetsfiles[:maxpacketfiles]

    return packetsfiles
<commit_msg>Detect bad GZIP packets files<commit_after>#!/usr/bin/env python3

# import math
import glob
import gzip
import multiprocessing
import sys
from pathlib import Path

# import matplotlib.patches as mpatches
# import numpy as np
import pandas as pd
from astropy import constants as const
from astropy import units as u

# from collections import namedtuple
from functools import lru_cache

import artistools as at

columns = (
    'number',
    'where',
    'type_id',
    'posx', 'posy', 'posz',
    'dirx', 'diry', 'dirz',
    'last_cross',
    'tdecay',
    'e_cmf',
    'e_rf',
    'nu_cmf',
    'nu_rf',
    'escape_type_id',
    'escape_time',
    'scat_count',
    'next_trans',
    'interactions',
    'last_event',
    'emissiontype',
    'trueemissiontype',
    'em_posx', 'em_posy', 'em_posz',
    'absorption_type',
    'absorption_freq',
    'nscatterings',
    'em_time',
    'absorptiondirx',
    'absorptiondiry',
    'absorptiondirz', 'stokes1', 'stokes2', 'stokes3', 'pol_dirx', 'pol_diry',
    'pol_dirz',
    'originated_from_positron',
    'true_emission_velocity',
    'trueem_time',
)

types = {
    10: 'TYPE_GAMMA',
    11: 'TYPE_RPKT',
    20: 'TYPE_NTLEPTON',
    32: 'TYPE_ESCAPE',
}

type_ids = dict((v, k) for k, v in types.items())

def add_derived_columns(dfpackets, modelpath, colnames, allnonemptymgilist=None):
    if dfpackets.empty:
        return dfpackets

    colnames = at.makelist(colnames)

    def em_modelgridindex(packet):
        return at.get_mgi_of_velocity_kms(modelpath, packet.emission_velocity * u.cm.to('km'),
                                          mgilist=allnonemptymgilist)

    def emtrue_modelgridindex(packet):
        return at.get_mgi_of_velocity_kms(modelpath, packet.true_emission_velocity * u.cm.to('km'), mgilist=allnonemptymgilist)

    def em_timestep(packet):
        return at.get_timestep_of_timedays(modelpath, packet.em_time * u.s.to('day'))

    if 'emission_velocity' in colnames:
        dfpackets.eval(
            "emission_velocity = sqrt(em_posx ** 2 + em_posy ** 2 + em_posz ** 2) / em_time",
            inplace=True)

    if 'em_modelgridindex' in colnames:
        if 'emission_velocity' not in dfpackets.columns:
            dfpackets = add_derived_columns(dfpackets, modelpath, ['emission_velocity'],
                                            allnonemptymgilist=allnonemptymgilist)
        dfpackets['em_modelgridindex'] = dfpackets.apply(em_modelgridindex, axis=1)

    if 'emtrue_modelgridindex' in colnames:
        dfpackets['emtrue_modelgridindex'] = dfpackets.apply(emtrue_modelgridindex, axis=1)

    if 'em_timestep' in colnames:
        dfpackets['em_timestep'] = dfpackets.apply(em_timestep, axis=1)

    return dfpackets

@at.diskcache(savegzipped=True)
def readfile(packetsfile, type=None, escape_type=None):
    """Read a packet file into a pandas DataFrame."""
    filesize = Path(packetsfile).stat().st_size / 1024 / 1024

    print(f'Reading {packetsfile} ({filesize:.1f} MiB)', end='')
    try:
        inputcolumncount = len(pd.read_csv(packetsfile, nrows=1, delim_whitespace=True, header=None).columns)
        if inputcolumncount < 3:
            print("\nWARNING: packets file has no columns!")
            print(open(packetsfile, "r").readlines())

    except gzip.BadGzipFile:
        print(f"\nBad Gzip File: {packetsfile}")
        raise gzip.BadGzipFile

    # the packets file may have a truncated set of columns, but we assume that they
    # are only truncated, i.e. the columns with the same index have the same meaning
    usecols_nodata = [n for n in columns if columns.index(n) >= inputcolumncount]
    # usecols_actual = [n for n in columns if columns.index(n) < inputcolumncount]

    dfpackets = pd.read_csv(
        packetsfile, delim_whitespace=True,
        names=columns[:inputcolumncount], header=None)

    print(f' ({len(dfpackets):.1e} packets', end='')

    if escape_type is not None and escape_type != '' and escape_type != 'ALL':
        assert type is None or type == 'TYPE_ESCAPE'
        dfpackets.query(f'type_id == {type_ids["TYPE_ESCAPE"]} and escape_type_id == {type_ids[escape_type]}',
                        inplace=True)
        print(f', {len(dfpackets)} escaped as {escape_type})')
    elif type is not None and type != 'ALL' and type != '':
        dfpackets.query(f'type_id == {type_ids[type]}', inplace=True)
        print(f', {len(dfpackets)} with type {type})')
    else:
        print(')')

    # dfpackets['type'] = dfpackets['type_id'].map(lambda x: types.get(x, x))
    # dfpackets['escape_type'] = dfpackets['escape_type_id'].map(lambda x: types.get(x, x))

    if usecols_nodata:
        print(f'WARNING: no data in packets file for columns: {usecols_nodata}')
        for col in usecols_nodata:
            dfpackets[col] = float('NaN')

    # # neglect light travel time correction
    # dfpackets.eval("t_arrive_d = escape_time * @u.s.to('day')", inplace=True)

    dfpackets.eval(
        "t_arrive_d = (escape_time - "
        "(posx * dirx + posy * diry + posz * dirz) / @const.c.to('cm/s').value) * @u.s.to('day')", inplace=True)

    return dfpackets

@lru_cache(maxsize=16)
def get_packetsfilepaths(modelpath, maxpacketfiles=None):
    packetsfiles = sorted(
        glob.glob(str(Path(modelpath, 'packets00_*.out*'))) +
        glob.glob(str(Path(modelpath, 'packets', 'packets00_*.out*'))))
    if maxpacketfiles is not None and maxpacketfiles > 0 and len(packetsfiles) > maxpacketfiles:
        print(f'Using only the first {maxpacketfiles} of {len(packetsfiles)} packets files')
        packetsfiles = packetsfiles[:maxpacketfiles]

    return packetsfiles
<|endoftext|>