<commit_before># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_intfiles
# Purpose:      From Spidered pages found, identifies files of potential interest.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     06/04/2014
# Copyright:   (c) Steve Micallef 2014
# Licence:     GPL
# -------------------------------------------------------------------------------

import re
import urllib
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_intfiles(SpiderFootPlugin):
    """Interesting Files:Footprint:Crawling and Scanning:errorprone:Identifies potential files of interest, e.g. office documents, zip files."""

    # Default options
    opts = {
        'pages': 20,  # Number of search results pages to iterate
        'fileexts': ["doc", "docx", "ppt", "pptx", "pdf", 'xls', 'xlsx', 'zip']
    }

    # Option descriptions
    optdescs = {
        'fileexts': "File extensions of files you consider interesting."
    }

    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["LINKED_URL_INTERNAL"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["INTERESTING_FILE"]

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        if eventData in self.results:
            return None
        else:
            self.results[eventData] = True

        for fileExt in self.opts['fileexts']:
            if "." + fileExt.lower() in eventData.lower():
                evt = SpiderFootEvent("INTERESTING_FILE", eventData,
                                      self.__name__, event)
                self.notifyListeners(evt)

# End of sfp_intfiles class
<commit_msg>Removed old pages config reference.<commit_after># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_intfiles
# Purpose:      From Spidered pages found, identifies files of potential interest.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     06/04/2014
# Copyright:   (c) Steve Micallef 2014
# Licence:     GPL
# -------------------------------------------------------------------------------

import re
import urllib
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_intfiles(SpiderFootPlugin):
    """Interesting Files:Footprint:Crawling and Scanning:errorprone:Identifies potential files of interest, e.g. office documents, zip files."""

    # Default options
    opts = {
        'fileexts': ["doc", "docx", "ppt", "pptx", "pdf", 'xls', 'xlsx', 'zip']
    }

    # Option descriptions
    optdescs = {
        'fileexts': "File extensions of files you consider interesting."
    }

    results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.results = dict()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["LINKED_URL_INTERNAL"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["INTERESTING_FILE"]

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        if eventData in self.results:
            return None
        else:
            self.results[eventData] = True

        for fileExt in self.opts['fileexts']:
            if "." + fileExt.lower() in eventData.lower():
                evt = SpiderFootEvent("INTERESTING_FILE", eventData,
                                      self.__name__, event)
                self.notifyListeners(evt)

# End of sfp_intfiles class
<|endoftext|>


<commit_before>'''
	Test condition creation for dialplan
'''

import unittest
from wirecurly.dialplan import condition
from nose import tools

class AppMock(object):
	"""An application interface mockup"""
	def __init__(self):
		super(AppMock, self).__init__()
		self.app_name = 'test'
		self.data = ''

class testConditionCreation(unittest.TestCase):
	'''
		Test condition creation
	'''

	def setUp(self):
		'''
			Condition fixtures for tests
		'''

		self.cond = condition.Condition('destination_number','1000')

	def test_condition_dict_ok(self):
		'''
			Test that condition is properly serialized
		'''
		assert isinstance(self.cond.todict(), dict)

	def test_condition_action_dict_ok(self):
		'''
			Test that condition with an action is properly serialized
		'''
		self.cond.addAction('answer','')
		assert isinstance(self.cond.todict(), dict)

	def test_condition_anti_action_dict_ok(self):
		'''
			Test that condition with an anti-action is properly serialized
		'''
		self.cond.addAntiAction('answer','')
		assert isinstance(self.cond.todict(), dict)

	def test_condition_action_anti_action_dict_ok(self):
		'''
			Test that condition with an action and an anti-action is properly serialized
		'''
		self.cond.addAction('answer','')
		self.cond.addAntiAction('hangup','')
		assert isinstance(self.cond.todict(), dict)

	def test_adding_action(self):
		'''
			Test if an action is properly add to a condition
		'''
		self.cond.addAction('answer','')
		assert self.cond.existAction('answer','')

	def test_adding_anti_action(self):
		'''
			Test if an anti-action is properly add to a condition
		'''
		self.cond.addAntiAction('hangup','')
		assert self.cond.existAntiAction('hangup','')

	def test_adding_application(self):
		'''
			Test adding an application to a Condition
		'''
		self.cond.addApplication(AppMock())
		assert self.cond.existAction('test', '')

	@tools.raises(ValueError)
	def test_adding_existing_action(self):
		'''
			Test adding an existing action
		'''
		self.cond.addAction('answer','')
		self.cond.addAction('answer','')

<commit_msg>fix a test that is now invalid result due to requirements change.<commit_after>'''
	Test condition creation for dialplan
'''

import unittest
from wirecurly.dialplan import condition
from nose import tools

class AppMock(object):
	"""An application interface mockup"""
	def __init__(self):
		super(AppMock, self).__init__()
		self.app_name = 'test'
		self.data = ''

class testConditionCreation(unittest.TestCase):
	'''
		Test condition creation
	'''

	def setUp(self):
		'''
			Condition fixtures for tests
		'''

		self.cond = condition.Condition('destination_number','1000')

	def test_condition_dict_ok(self):
		'''
			Test that condition is properly serialized
		'''
		assert isinstance(self.cond.todict(), dict)

	def test_condition_action_dict_ok(self):
		'''
			Test that condition with an action is properly serialized
		'''
		self.cond.addAction('answer','')
		assert isinstance(self.cond.todict(), dict)

	def test_condition_anti_action_dict_ok(self):
		'''
			Test that condition with an anti-action is properly serialized
		'''
		self.cond.addAntiAction('answer','')
		assert isinstance(self.cond.todict(), dict)

	def test_condition_action_anti_action_dict_ok(self):
		'''
			Test that condition with an action and an anti-action is properly serialized
		'''
		self.cond.addAction('answer','')
		self.cond.addAntiAction('hangup','')
		assert isinstance(self.cond.todict(), dict)

	def test_adding_action(self):
		'''
			Test if an action is properly add to a condition
		'''
		self.cond.addAction('answer','')
		assert self.cond.existAction('answer','')

	def test_adding_anti_action(self):
		'''
			Test if an anti-action is properly add to a condition
		'''
		self.cond.addAntiAction('hangup','')
		assert self.cond.existAntiAction('hangup','')

	def test_adding_application(self):
		'''
			Test adding an application to a Condition
		'''
		self.cond.addApplication(AppMock())
		assert self.cond.existAction('test', '')

	def test_adding_existing_action(self):
		'''
			Test adding an existing action
		'''
		self.cond.addAction('answer','')
		self.cond.addAction('answer','')

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""The code."""

class User(object):

    """The user model."""

    def __init__(self, id):
        """Create a new user with the given id."""
        self.id = id
        self.adjacent = set()
        self.new_version = False

    def connect(self, other):
        """Add a new relationship."""
        self.adjacent.add(other)
        other.adjacent.add(self)

def total_infection(user):
    """Infect all the users connected to user.

    This is implemented with a breadth-first search.
    """
    if user.new_version:
        return
    user.new_version = True
    for adjacent in user.adjacent:
        total_infection(adjacent)

def main():
    """The main function."""

if __name__ == '__main__':
    main()
<commit_msg>I must not tell lies.<commit_after># -*- coding: utf-8 -*-
"""The code."""

class User(object):

    """The user model."""

    def __init__(self, id):
        """Create a new user with the given id."""
        self.id = id
        self.adjacent = set()
        self.new_version = False

    def connect(self, other):
        """Add a new relationship."""
        self.adjacent.add(other)
        other.adjacent.add(self)

def total_infection(user):
    """Infect all the users connected to user.

    This is implemented with a depth-first search.
    """
    if user.new_version:
        return
    user.new_version = True
    for adjacent in user.adjacent:
        total_infection(adjacent)

def main():
    """The main function."""

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import pymongo
import numpy as np
import sys

db = pymongo.Connection()['ieqa-dev']
test_name = sys.argv[1]
test_method = '%s_method' % test_name

test_methods = set()
for r in db.results.find():
    if r[test_method]:
        test_methods.add(r[test_method])

test_values = {}
for m in test_methods:
    tests = []
    for r in db.results.find({test_method: m}):
        if r[test_name]:
            tests.append(r[test_name])
    test_values[m] = tests

all_methods = []

def calculate(values, principle):
    mean = np.mean(values)
    std = np.std(values)
    print '\n' + principle
    print '-----------------'
    print 'N = %d' % (len(values))
    print "mean = {0:f}".format(mean)
    print "SD = {0:f}".format(std)
    print "%CV = {0:f}".format(std / mean * 100.0)
    print "mean + 1.5SD = {0:f}".format(mean + 1.5 * std)
    print "mean - 1.5SD = {0:f}".format(mean - 1.5 * std)

for m in test_methods:
    calculate(test_values[m], m)
    all_methods += test_values[m]

principle = "All"

calculate(all_methods, principle)
<commit_msg>added filter_values function and did minor refactoring<commit_after>import pymongo
import numpy as np
import sys

db = pymongo.Connection()['ieqa-dev']
test_name = sys.argv[1]
test_method = '%s_method' % test_name

test_methods = set()
for r in db.results.find():
    if r[test_method]:
        test_methods.add(r[test_method])

test_values = {}
for m in test_methods:
    tests = []
    for r in db.results.find({test_method: m}):
        if r[test_name]:
            tests.append(r[test_name])
    test_values[m] = tests

all_methods = []

def calculate(values, principle, sd_range):
    mean = np.mean(values)
    std = np.std(values)
    print '\n' + principle
    print '-----------------'
    print 'N = %d' % (len(values))
    print "mean = {0:f}".format(mean)
    print "SD = {0:f}".format(std)
    print "%CV = {0:f}".format(std / mean * 100.0)
    print "mean + {0:f}SD = {1:f}".format(sd_range, mean + 1.5 * std)
    print "mean - {0:f}SD = {1:f}".format(sd_range, mean - 1.5 * std)

def filter_values(values, sd_range):
    """Returns values within std range"""
    mean = np.mean(values)
    std = np.std(values)
    upper = mean + (sd_range * std)
    lower = mean - (sd_range * std)
    filtered_values = []
    for val in values:
        if upper >= val >= lower:
            filtered_values.append(val)

    return filtered_values

for m in test_methods:
    calculate(test_values[m], m, 1.5)
    all_methods += test_values[m]

principle = "All"

calculate(all_methods, principle, 1.5)

for m in test_methods:
    filtered_values = filter_values(test_values[m], 3)
    calculate(filtered_values, 'New ' + m, 1.5)

filtered_values = filter_values(all_methods, 3)
calculate(filtered_values, 'New all', 1.5)<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Script to compact all Brython scripts in a single one."""

import time
import datetime
import os
import re
import sys
import tarfile
import zipfile
import shutil

import javascript_minifier

if sys.version_info[0] != 3:
    raise ValueError("This script only works with Python 3")

# path of parent directory
pdir = os.path.dirname(os.getcwd())
# version info
version = [3, 8, 0, "final", 0]
implementation = [3, 8, 0, "dev", 0]

# version name
vname = '.'.join(str(x) for x in implementation[:3])
if implementation[3] == 'rc':
    vname += 'rc%s' % implementation[4]

def run():
    # update package.json
    package_file = os.path.join(pdir, 'npm', 'package.json')
    with open(package_file, encoding="utf-8") as fobj:
        package_info = fobj.read()
        package_info = re.sub('"version": "(.*)"',
            '"version": "{}"'.format(vname),
            package_info)

    with open(package_file, "w", encoding="utf-8") as fobj:
        fobj.write(package_info)

    abs_path = lambda _pth: os.path.join(os.path.dirname(os.getcwd()), 'www',
        'src', _pth)
    now = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')

    timestamp = int(1000 * time.time())

    # update version number
    with open(abs_path('version_info.js'), 'w') as vinfo_file_out:
        # implementation[2] = now
        vinfo_file_out.write('__BRYTHON__.implementation = %s\n'
            %implementation)
        vinfo_file_out.write('__BRYTHON__.__MAGIC__ = "%s"\n' %
            '.'.join(['%s' % _i for _i in implementation[:3]]))
        vinfo_file_out.write('__BRYTHON__.version_info = %s\n' % str(version))
        vinfo_file_out.write('__BRYTHON__.compiled_date = "%s"\n'
            %str(datetime.datetime.now()))
        vinfo_file_out.write('__BRYTHON__.timestamp = {}\n'.format(timestamp))
        # builtin module names = list of scripts in src/libs
        vinfo_file_out.write('__BRYTHON__.builtin_module_names = [')
        _modules = ['"%s"' % fname.split('.')[0]
                   for fname in os.listdir(abs_path('libs'))
                   if fname.endswith('.js')]

        # Sort modules so that git diff's don't change between runs
        _modules.sort()
        vinfo_file_out.write(',\n    '.join(_modules))
        vinfo_file_out.write(']\n')

    # generate html files that compare Brython and CPython distributions
    import make_stdlib_list

    import make_stdlib_static
    # build brython.js from base Javascript files
    sources = [
        'brython_builtins', 'version_info', 'py2js', 'loaders',
        'py_object', 'py_type', 'py_utils', 'py_builtin_functions',
        'py_exceptions', 'py_range_slice', 'py_bytes', 'py_set', 'js_objects',
        'stdlib_paths', 'py_import', 'py_float', 'py_int', 'py_long_int',
        'py_complex', 'py_sort', 'py_list', 'unicode_data', 'py_string',
        'py_dict', 'py_dom', 'py_generator', 'builtin_modules',
        'py_import_hooks', 'async'
    ]

    res = """// brython.js brython.info
// version {}
// implementation {}
// version compiled from commented, indented source files at
// github.com/brython-dev/brython
""".format(version, implementation)

    res_no_static = res # same but without static stdlib files
    src_size = 0

    for fname in sources:
        src = open(abs_path(fname)+'.js').read() + '\n'
        src_size += len(src)
        mini = javascript_minifier.minify(src) + ";\n"
        res += mini
        if fname == 'stdlib_paths':
            res_no_static += "__BRYTHON__.stdlib = {}\n"
        else:
            res_no_static += mini

    res = re.sub(r'\bcontext\b', 'C', res)

    with open(abs_path('brython.js'), 'w', newline="\n") as out:
        out.write(res)

    with open(abs_path('brython_no_static.js'), 'w', newline="\n") as out:
        out.write(res_no_static)

    print(('size : originals %s compact %s gain %.2f' %
          (src_size, len(res), 100 * (src_size - len(res)) / src_size)))

    sys.path.append("scripts")

    try:
        import make_VFS
    except ImportError:
        print("Cannot find make_VFS, so we won't make brython_stdlib.js.js")
        make_VFS = None
        sys.exit()

    make_VFS.process(os.path.join(pdir, 'www', 'src', 'brython_stdlib.js'))

    # Create brython_dist.js : core + libraries
    src_dir = os.path.join(pdir, 'www', 'src')
    with open(os.path.join(src_dir, 'brython_dist.js'), 'w') as distrib:
        distrib.write(open(os.path.join(src_dir, 'brython.js')).read())
        distrib.write(open(os.path.join(src_dir, 'brython_stdlib.js')).read())

    # update implementation in brython.py
    br_script = os.path.join(pdir, 'setup', 'brython.py')
    with open(br_script, encoding='utf-8') as f:
        br = f.read()
        mo = re.search('^implementation = "(.*)"$', br, flags=re.M)
        if not mo:
            raise Exception("missing implementation in brython.py")
        br_versioned = re.sub('^implementation = "(.*)"$',
                              'implementation = "{}"'.format(vname),
                              br,
                              flags=re.M)

    with open(br_script, "w", encoding="utf-8") as out:
        out.write(br_versioned)

    # copy files in folder /setup
    sdir = os.path.join(pdir, 'setup', 'data')

    for f in ['brython_stdlib.js', 'unicode.txt']:
        shutil.copyfile(os.path.join(src_dir, f), os.path.join(sdir, f))

    # copy the version of brython.js that has no static stdlib
    shutil.copyfile(os.path.join(src_dir, "brython_no_static.js"),
        os.path.join(sdir, "brython.js"))

    # copy demo.html
    with open(os.path.join(pdir, 'www', 'demo.html'), encoding="utf-8") as f:
        demo = f.read()
    start_tag = "<!-- start copy -->"
    end_tag = "<!-- end copy -->"
    start = demo.find(start_tag)
    if start == -1:
        raise Exception("No tag <!-- start copy --> in demo.html")
    end = demo.find(end_tag)
    if end == -1:
        raise Exception("No tag <!-- end copy --> in demo.html")
    body = demo[start + len(start_tag) : end].strip()

    with open(os.path.join(sdir, "demo.tmpl"), encoding="utf-8") as f:
        template = f.read()

    demo = template.replace("{{body}}", body)

    with open(os.path.join(sdir, "demo.html"), "w", encoding="utf-8") as out:
        out.write(demo)

    # copy files in folder /npm
    npmdir = os.path.join(pdir, 'npm')
    for f in ['brython.js', 'brython_stdlib.js', 'unicode.txt']:
        shutil.copyfile(os.path.join(src_dir, f), os.path.join(npmdir, f))

    # create zip files
    print('create zip files in /setup/data')
    name = 'Brython-{}'.format(vname)
    dest_path = os.path.join(sdir, name)
    dist1 = tarfile.open(dest_path + '.tar.gz', mode='w:gz')
    dist2 = tarfile.open(dest_path+'.tar.bz2', mode='w:bz2')
    dist3 = zipfile.ZipFile(dest_path + '.zip', mode='w',
                            compression=zipfile.ZIP_DEFLATED)

    paths = ['README.txt', 'demo.html',
        'brython.js', 'brython_stdlib.js', 'unicode.txt']

    for arc, wfunc in ((dist1, dist1.add), (dist2, dist2.add),
            (dist3, dist3.write)):
        for path in paths:
            wfunc(os.path.join(sdir, path),
                arcname=os.path.join(name, path))

        arc.close()

    # changelog file
    print('write changelog file')
    try:
        first = 'Changes in Brython version {}'.format(vname)
        with open(os.path.join(pdir, 'setup', 'changelog.txt')) as f:
            input_changelog_data_string = f.read()
        with open(os.path.join(pdir, 'setup', 'data',
                'changelog_{}.txt'.format(vname)), 'w') as out:
            out.write('%s\n' % first)
            out.write('%s\n\n' % ('=' * len(first)))
            out.write(input_changelog_data_string)
    except Exception as error:
        print(error)
        print("Warning - no changelog file")

if __name__ == "__main__":
    run()
<commit_msg>Update make_dist.py for the new directory structure<commit_after># -*- coding: utf-8 -*-

"""Script to compact all Brython scripts in a single one."""

import time
import datetime
import os
import re
import sys
import tarfile
import zipfile
import shutil

import javascript_minifier

if sys.version_info[0] != 3:
    raise ValueError("This script only works with Python 3")

# path of parent directory
pdir = os.path.dirname(os.getcwd())
# version info
version = [3, 8, 0, "final", 0]
implementation = [3, 8, 0, "dev", 0]

# version name
vname = '.'.join(str(x) for x in implementation[:3])
if implementation[3] == 'rc':
    vname += 'rc%s' % implementation[4]

def run():
    # update package.json
    package_file = os.path.join(pdir, 'npm', 'package.json')
    with open(package_file, encoding="utf-8") as fobj:
        package_info = fobj.read()
        package_info = re.sub('"version": "(.*)"',
            '"version": "{}"'.format(vname),
            package_info)

    with open(package_file, "w", encoding="utf-8") as fobj:
        fobj.write(package_info)

    abs_path = lambda _pth: os.path.join(os.path.dirname(os.getcwd()), 'www',
        'src', _pth)
    now = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')

    timestamp = int(1000 * time.time())

    # update version number
    with open(abs_path('version_info.js'), 'w') as vinfo_file_out:
        # implementation[2] = now
        vinfo_file_out.write('__BRYTHON__.implementation = %s\n'
            %implementation)
        vinfo_file_out.write('__BRYTHON__.__MAGIC__ = "%s"\n' %
            '.'.join(['%s' % _i for _i in implementation[:3]]))
        vinfo_file_out.write('__BRYTHON__.version_info = %s\n' % str(version))
        vinfo_file_out.write('__BRYTHON__.compiled_date = "%s"\n'
            %str(datetime.datetime.now()))
        vinfo_file_out.write('__BRYTHON__.timestamp = {}\n'.format(timestamp))
        # builtin module names = list of scripts in src/libs
        vinfo_file_out.write('__BRYTHON__.builtin_module_names = [')
        _modules = ['"%s"' % fname.split('.')[0]
                   for fname in os.listdir(abs_path('libs'))
                   if fname.endswith('.js')]

        # Sort modules so that git diff's don't change between runs
        _modules.sort()
        vinfo_file_out.write(',\n    '.join(_modules))
        vinfo_file_out.write(']\n')

    # generate html files that compare Brython and CPython distributions
    import make_stdlib_list

    import make_stdlib_static
    # build brython.js from base Javascript files
    sources = [
        'brython_builtins', 'version_info', 'py2js', 'loaders',
        'py_object', 'py_type', 'py_utils', 'py_builtin_functions',
        'py_exceptions', 'py_range_slice', 'py_bytes', 'py_set', 'js_objects',
        'stdlib_paths', 'py_import', 'py_float', 'py_int', 'py_long_int',
        'py_complex', 'py_sort', 'py_list', 'unicode_data', 'py_string',
        'py_dict', 'py_dom', 'py_generator', 'builtin_modules',
        'py_import_hooks', 'async'
    ]

    res = """// brython.js brython.info
// version {}
// implementation {}
// version compiled from commented, indented source files at
// github.com/brython-dev/brython
""".format(version, implementation)

    res_no_static = res # same but without static stdlib files
    src_size = 0

    for fname in sources:
        src = open(abs_path(fname)+'.js').read() + '\n'
        src_size += len(src)
        mini = javascript_minifier.minify(src) + ";\n"
        res += mini
        if fname == 'stdlib_paths':
            res_no_static += "__BRYTHON__.stdlib = {}\n"
        else:
            res_no_static += mini

    res = re.sub(r'\bcontext\b', 'C', res)

    with open(abs_path('brython.js'), 'w', newline="\n") as out:
        out.write(res)

    with open(abs_path('brython_no_static.js'), 'w', newline="\n") as out:
        out.write(res_no_static)

    print(('size : originals %s compact %s gain %.2f' %
          (src_size, len(res), 100 * (src_size - len(res)) / src_size)))

    sys.path.append("scripts")

    try:
        import make_VFS
    except ImportError:
        print("Cannot find make_VFS, so we won't make brython_stdlib.js.js")
        make_VFS = None
        sys.exit()

    make_VFS.process(os.path.join(pdir, 'www', 'src', 'brython_stdlib.js'))

    # Create brython_dist.js : core + libraries
    src_dir = os.path.join(pdir, 'www', 'src')
    with open(os.path.join(src_dir, 'brython_dist.js'), 'w') as distrib:
        distrib.write(open(os.path.join(src_dir, 'brython.js')).read())
        distrib.write(open(os.path.join(src_dir, 'brython_stdlib.js')).read())

    # update implementation in brython.py
    br_script = os.path.join(pdir, 'setup', 'brython.py')
    with open(br_script, encoding='utf-8') as f:
        br = f.read()
        mo = re.search('^implementation = "(.*)"$', br, flags=re.M)
        if not mo:
            raise Exception("missing implementation in brython.py")
        br_versioned = re.sub('^implementation = "(.*)"$',
                              'implementation = "{}"'.format(vname),
                              br,
                              flags=re.M)

    with open(br_script, "w", encoding="utf-8") as out:
        out.write(br_versioned)

    # copy demo.html
    with open(os.path.join(pdir, 'www', 'demo.html'), encoding="utf-8") as f:
        demo = f.read()
    start_tag = "<!-- start copy -->"
    end_tag = "<!-- end copy -->"
    start = demo.find(start_tag)
    if start == -1:
        raise Exception("No tag <!-- start copy --> in demo.html")
    end = demo.find(end_tag)
    if end == -1:
        raise Exception("No tag <!-- end copy --> in demo.html")
    body = demo[start + len(start_tag) : end].strip()

    release_dir = os.path.join(pdir, "releases")
    with open(os.path.join(release_dir, "demo.tmpl"), encoding="utf-8") as f:
        template = f.read()

    demo = template.replace("{{body}}", body)

    with open(os.path.join(release_dir, "demo.html"),
            "w", encoding="utf-8") as out:
        out.write(demo)

    # copy files in folder /npm
    npmdir = os.path.join(pdir, 'npm')
    for f in ['brython.js', 'brython_stdlib.js', 'unicode.txt']:
        shutil.copyfile(os.path.join(src_dir, f), os.path.join(npmdir, f))

    # create zip files
    print('create zip files in /releases')
    name = 'Brython-{}'.format(vname)
    dest_path = os.path.join(release_dir, name)
    dist1 = tarfile.open(dest_path + '.tar.gz', mode='w:gz')
    dist2 = tarfile.open(dest_path+'.tar.bz2', mode='w:bz2')
    dist3 = zipfile.ZipFile(dest_path + '.zip', mode='w',
                            compression=zipfile.ZIP_DEFLATED)

    paths1 = ['README.txt', 'demo.html', 'index.html']
    paths2 = ['brython.js', 'brython_stdlib.js', 'unicode.txt']

    for arc, wfunc in ((dist1, dist1.add), (dist2, dist2.add),
            (dist3, dist3.write)):
        for path in paths1:
            wfunc(os.path.join(release_dir, path),
                arcname=os.path.join(name, path))
        for path in paths2:
            wfunc(abs_path(path),
                arcname=os.path.join(name, path))

        arc.close()

    # changelog file
    print('write changelog file')
    try:
        first = 'Changes in Brython version {}'.format(vname)
        with open(os.path.join(pdir, 'setup', 'changelog.txt')) as f:
            input_changelog_data_string = f.read()
        with open(os.path.join(pdir, 'setup', 'data',
                'changelog_{}.txt'.format(vname)), 'w') as out:
            out.write('%s\n' % first)
            out.write('%s\n\n' % ('=' * len(first)))
            out.write(input_changelog_data_string)
    except Exception as error:
        print(error)
        print("Warning - no changelog file")

if __name__ == "__main__":
    run()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Use URL to EUPS candidate tag file to git tag repos with official version
"""

# Technical Debt
# --------------
# - sort out the certificate so we don't have to supress warnings
# - completely hide eups-specifics from this file
# - skips non-github repos - can add repos.yaml knowhow to address this
# - worth doing the smart thing for externals? (yes for Sims)
# - deal with authentication version

# Known Bugs
# ----------
# Yeah, the candidate logic is broken, will fix

from __future__ import print_function

import logging
import os
import sys
import argparse
import textwrap
import string
from datetime import datetime
from getpass import getuser
import urllib3
from .. import codetools

def parse_args():
    """Parse command-line arguments"""
    user = getuser()

    parser = argparse.ArgumentParser(
        prog='github-tag-version',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""

        Tag all repositories in a GitHub org using a team-based scheme

        Examples:
        github-tag-version --org lsst --team 'Data Management' w.2015.33 b1630

        github-tag-version --org lsst --team 'Data Management' \
            --team 'External' --candidate v11_0_rc2 11.0.rc2 b1679

        Note that the access token must have access to these oauth scopes:
            * read:org
            * repo

        The token generated by `github-auth --user` should have sufficient
        permissions.
        """),
        epilog='Part of codekit: https://github.com/lsst-sqre/sqre-codekit'
    )

    # for safety, default to dummy org <user>-shadow
    # will fail for most people but see github_fork_repos in this module
    # on how to get your own

    parser.add_argument('tag')
    parser.add_argument('manifest')
    parser.add_argument(
        '--org',
        default=user + '-shadow')
    parser.add_argument(
        '--team',
        action='append',
        required=True,
        help="team whose repos may be tagged (can specify several times")
    parser.add_argument('--candidate')
    parser.add_argument('--dry-run', action='store_true')
    parser.add_argument(
        '--tagger',
        help='Name of person making the tag - defaults to gitconfig value')
    parser.add_argument(
        '--email',
        help='Email address of tagger - defaults to gitconfig value')
    parser.add_argument(
        '--token-path',
        default='~/.sq_github_token_delete',
        help='Use a token (made with github-auth) in a non-standard location')
    parser.add_argument(
        '--token',
        default=None,
        help='Literal github personal access token string')
    parser.add_argument(
        '-d', '--debug',
        action='store_true',
        default=os.getenv('DM_SQUARE_DEBUG'),
        help='Debug mode')
    parser.add_argument('-v', '--version',
                        action='version', version='%(prog)s 0.5')
    return parser.parse_args()

def main():
    """Create the tag"""
    # pylint: disable=too-many-locals,too-many-nested-blocks,too-many-branches
    # pylint: disable=too-many-statements
    # Although maybe that is a hint that we should break this up...
    args = parse_args()

    orgname = args.org
    version = args.tag

    # if email not specified, try getting it from the gitconfig
    email = args.email
    if email is None:
        email = codetools.gituseremail()
        if email is None:
            sys.exit("Specify --email option")
    if args.debug:
        print("email is " + email)
    # ditto for the name of the tagger
    tagger = args.tagger
    if tagger is None:
        tagger = codetools.gitusername()
        if tagger is None:
            sys.exit("Specify --tagger option")
    if args.debug:
        print("tagger name is " + tagger)

    # The candidate is assumed to be the requested EUPS tag unless
    # otherwise specified with the --candidate option The reason to
    # currently do this is that for weeklies and other internal builds,
    # it's okay to eups publish the weekly and git tag post-facto. However
    # for official releases, we don't want to publish until the git tag
    # goes down, because we want to eups publish the build that has the
    # official versions in the eups ref.

    if args.candidate:
        candidate = args.candidate
    else:
        candidate = args.tag

    eupsbuild = args.manifest  # sadly we need to "just" know this
    message_template = 'Version {v} release from {c}/{b}'
    message = message_template.format(v=version, c=candidate, b=eupsbuild)
    eupspkg_site = 'https://eups.lsst.codes/stack/src/'

    # generate timestamp for github API
    now = datetime.utcnow()
    timestamp = now.isoformat()[0:19] + 'Z'
    if args.debug:
        print(timestamp)

    tagstuff = dict(name=tagger,
                    email=email,
                    date=timestamp)

    if args.debug:
        print(tagstuff)

    ghb = codetools.login_github(token_path=args.token_path, token=args.token)
    if args.debug:
        print(type(ghb))

    if args.debug:
        print("Tagging repos in ", orgname)

    # generate eups-style version
    # eups no likey semantic versioning markup, wants underscores

    # Python 2/3 compatibility
    try:
        # Python 2
        cmap = string.maketrans('.-', '__')
    except AttributeError:
        # Python 3
        cmap = str.maketrans('.-', '__')  # pylint: disable=no-member

    # eups_version = version.translate(map)
    eups_candidate = candidate.translate(cmap)

    # construct url
    eupspkg_taglist = '/'.join((eupspkg_site, 'tags',
                                eups_candidate + '.list'))
    if args.debug:
        print(eupspkg_taglist)

    http = urllib3.poolmanager.PoolManager()

    # supress the certificate warning - technical debt
    urllib3.disable_warnings()  # NOQA
    if args.debug:
        # pylint: disable=fixme
        # FIXME what's going on here? assigning a logger to a package?
        logging.getLogger('requests.packages.urllib3')  # NOQA
        stream_handler = logging.StreamHandler()
        logger = logging.getLogger('github3')
        logger.addHandler(stream_handler)
        logger.setLevel(logging.DEBUG)

    manifest = http.request('GET', eupspkg_taglist)

    if manifest.status >= 300:
        sys.exit("Failed GET")

    entries = manifest.data.splitlines()

    for entry in entries:
        # Python 2/3 accomodation
        if not isinstance(entry, str):
            entry = str(entry, 'utf-8')
        # skip commented out and blank lines
        if entry.startswith('#'):
            continue
        if entry.startswith('EUPS'):
            continue
        if entry == '':
            continue

        # extract the repo and eups tag from the entry
        (upstream, _, eups_tag) = entry.split()
        if args.debug:
            print(upstream, eups_tag)

        # okay so we still have the data dirs on gitolite
        # for now, just skip them and record them.
        # question is should they be on different tagging scheme anyway?
        # at this point these are: afwdata, astrometry_net_data qserv_testdata

        repo = ghb.repository(orgname, upstream)

        # if the repo is not in github skip it for now
        # see TD
        if not hasattr(repo, 'name'):
            print('!!! SKIPPING', upstream, (60 - len(upstream)) * '-')
            continue

        if not sum(1 for _ in repo.teams()):
            print('!!! repo has NO teams -- SKIPPING', upstream,
                  (45 - len(upstream)) * '-')
            continue

        for team in repo.teams():
            if team.name in args.team:
                if args.debug or args.dry_run:
                    print(repo.name.ljust(40), 'found in', team.name)
                sha = codetools.eups2git_ref(eups_ref=eups_tag,
                                             repo=repo.name,
                                             eupsbuild=eupsbuild,
                                             debug=args.debug)
                if args.debug or args.dry_run:
                    print('Will tag sha: {sha} as {v} (was {t})'.format(
                        sha=sha, v=version, t=eups_tag))

                if not args.dry_run:
                    try:
                        # create_tag() returns a Tag object on success or None
                        # on failure
                        tag = repo.create_tag(tag=version,
                                              message=message,
                                              sha=sha,
                                              obj_type='commit',
                                              tagger=tagstuff,
                                              lightweight=False)
                        if tag is None:
                            raise RuntimeError('failed to create git tag')

                    except Exception as exc:  # pylint: disable=broad-except
                        print('OOPS: -------------------')
                        print(str(exc))
                        print('OOPS: -------------------')

            else:
                if args.debug:
                    print('No action for', repo.name,
                          'belonging to', team.name)

if __name__ == '__main__':
    main()
<commit_msg>add --force-tag parameter<commit_after>#!/usr/bin/env python
"""
Use URL to EUPS candidate tag file to git tag repos with official version
"""

# Technical Debt
# --------------
# - sort out the certificate so we don't have to supress warnings
# - completely hide eups-specifics from this file
# - skips non-github repos - can add repos.yaml knowhow to address this
# - worth doing the smart thing for externals? (yes for Sims)
# - deal with authentication version

# Known Bugs
# ----------
# Yeah, the candidate logic is broken, will fix

from __future__ import print_function

import logging
import os
import sys
import argparse
import textwrap
import string
from datetime import datetime
from getpass import getuser
import urllib3
from .. import codetools

def parse_args():
    """Parse command-line arguments"""
    user = getuser()

    parser = argparse.ArgumentParser(
        prog='github-tag-version',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""

        Tag all repositories in a GitHub org using a team-based scheme

        Examples:
        github-tag-version --org lsst --team 'Data Management' w.2015.33 b1630

        github-tag-version --org lsst --team 'Data Management' \
            --team 'External' --candidate v11_0_rc2 11.0.rc2 b1679

        Note that the access token must have access to these oauth scopes:
            * read:org
            * repo

        The token generated by `github-auth --user` should have sufficient
        permissions.
        """),
        epilog='Part of codekit: https://github.com/lsst-sqre/sqre-codekit'
    )

    # for safety, default to dummy org <user>-shadow
    # will fail for most people but see github_fork_repos in this module
    # on how to get your own

    parser.add_argument('tag')
    parser.add_argument('manifest')
    parser.add_argument(
        '--org',
        default=user + '-shadow')
    parser.add_argument(
        '--team',
        action='append',
        required=True,
        help="team whose repos may be tagged (can specify several times")
    parser.add_argument('--candidate')
    parser.add_argument('--dry-run', action='store_true')
    parser.add_argument(
        '--tagger',
        help='Name of person making the tag - defaults to gitconfig value')
    parser.add_argument(
        '--email',
        help='Email address of tagger - defaults to gitconfig value')
    parser.add_argument(
        '--token-path',
        default='~/.sq_github_token_delete',
        help='Use a token (made with github-auth) in a non-standard location')
    parser.add_argument(
        '--token',
        default=None,
        help='Literal github personal access token string')
    parser.add_argument(
        '--force-tag',
        action='store_true',
        help='Force moving pre-existing annotated git tags.')
    parser.add_argument(
        '-d', '--debug',
        action='store_true',
        default=os.getenv('DM_SQUARE_DEBUG'),
        help='Debug mode')
    parser.add_argument('-v', '--version',
                        action='version', version='%(prog)s 0.5')
    return parser.parse_args()

def main():
    """Create the tag"""
    # pylint: disable=too-many-locals,too-many-nested-blocks,too-many-branches
    # pylint: disable=too-many-statements
    # Although maybe that is a hint that we should break this up...
    args = parse_args()

    orgname = args.org
    version = args.tag

    # if email not specified, try getting it from the gitconfig
    email = args.email
    if email is None:
        email = codetools.gituseremail()
        if email is None:
            sys.exit("Specify --email option")
    if args.debug:
        print("email is " + email)
    # ditto for the name of the tagger
    tagger = args.tagger
    if tagger is None:
        tagger = codetools.gitusername()
        if tagger is None:
            sys.exit("Specify --tagger option")
    if args.debug:
        print("tagger name is " + tagger)

    # The candidate is assumed to be the requested EUPS tag unless
    # otherwise specified with the --candidate option The reason to
    # currently do this is that for weeklies and other internal builds,
    # it's okay to eups publish the weekly and git tag post-facto. However
    # for official releases, we don't want to publish until the git tag
    # goes down, because we want to eups publish the build that has the
    # official versions in the eups ref.

    if args.candidate:
        candidate = args.candidate
    else:
        candidate = args.tag

    eupsbuild = args.manifest  # sadly we need to "just" know this
    message_template = 'Version {v} release from {c}/{b}'
    message = message_template.format(v=version, c=candidate, b=eupsbuild)
    eupspkg_site = 'https://eups.lsst.codes/stack/src/'

    # generate timestamp for github API
    now = datetime.utcnow()
    timestamp = now.isoformat()[0:19] + 'Z'
    if args.debug:
        print(timestamp)

    tagstuff = dict(name=tagger,
                    email=email,
                    date=timestamp)

    if args.debug:
        print(tagstuff)

    ghb = codetools.login_github(token_path=args.token_path, token=args.token)
    if args.debug:
        print(type(ghb))

    if args.debug:
        print("Tagging repos in ", orgname)

    # generate eups-style version
    # eups no likey semantic versioning markup, wants underscores

    # Python 2/3 compatibility
    try:
        # Python 2
        cmap = string.maketrans('.-', '__')
    except AttributeError:
        # Python 3
        cmap = str.maketrans('.-', '__')  # pylint: disable=no-member

    # eups_version = version.translate(map)
    eups_candidate = candidate.translate(cmap)

    # construct url
    eupspkg_taglist = '/'.join((eupspkg_site, 'tags',
                                eups_candidate + '.list'))
    if args.debug:
        print(eupspkg_taglist)

    http = urllib3.poolmanager.PoolManager()

    # supress the certificate warning - technical debt
    urllib3.disable_warnings()  # NOQA
    if args.debug:
        # pylint: disable=fixme
        # FIXME what's going on here? assigning a logger to a package?
        logging.getLogger('requests.packages.urllib3')  # NOQA
        stream_handler = logging.StreamHandler()
        logger = logging.getLogger('github3')
        logger.addHandler(stream_handler)
        logger.setLevel(logging.DEBUG)

    manifest = http.request('GET', eupspkg_taglist)

    if manifest.status >= 300:
        sys.exit("Failed GET")

    entries = manifest.data.splitlines()

    for entry in entries:
        # Python 2/3 accomodation
        if not isinstance(entry, str):
            entry = str(entry, 'utf-8')
        # skip commented out and blank lines
        if entry.startswith('#'):
            continue
        if entry.startswith('EUPS'):
            continue
        if entry == '':
            continue

        # extract the repo and eups tag from the entry
        (upstream, _, eups_tag) = entry.split()
        if args.debug:
            print(upstream, eups_tag)

        # okay so we still have the data dirs on gitolite
        # for now, just skip them and record them.
        # question is should they be on different tagging scheme anyway?
        # at this point these are: afwdata, astrometry_net_data qserv_testdata

        repo = ghb.repository(orgname, upstream)

        # if the repo is not in github skip it for now
        # see TD
        if not hasattr(repo, 'name'):
            print('!!! SKIPPING', upstream, (60 - len(upstream)) * '-')
            continue

        if not sum(1 for _ in repo.teams()):
            print('!!! repo has NO teams -- SKIPPING', upstream,
                  (45 - len(upstream)) * '-')
            continue

        for team in repo.teams():
            if team.name in args.team:
                if args.debug or args.dry_run:
                    print(repo.name.ljust(40), 'found in', team.name)
                sha = codetools.eups2git_ref(eups_ref=eups_tag,
                                             repo=repo.name,
                                             eupsbuild=eupsbuild,
                                             debug=args.debug)
                if args.debug or args.dry_run:
                    print('Will tag sha: {sha} as {v} (was {t})'.format(
                        sha=sha, v=version, t=eups_tag))

                if not args.dry_run:
                    try:
                        # create_tag() returns a Tag object on success or None
                        # on failure
                        tag = repo.create_tag(tag=version,
                                              message=message,
                                              sha=sha,
                                              obj_type='commit',
                                              tagger=tagstuff,
                                              lightweight=False,
                                              update=args.force_tag)
                        if tag is None:
                            raise RuntimeError('failed to create git tag')

                    except Exception as exc:  # pylint: disable=broad-except
                        print('OOPS: -------------------')
                        print(str(exc))
                        print('OOPS: -------------------')

            else:
                if args.debug:
                    print('No action for', repo.name,
                          'belonging to', team.name)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_s3bucket
# Purpose:      SpiderFoot plug-in for identifying potential S3 buckets related to
#               the target.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     24/07/2016
# Copyright:   (c) Steve Micallef 2016
# Licence:     GPL
# -------------------------------------------------------------------------------

import threading
import time
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_s3bucket(SpiderFootPlugin):
    """Amazon S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential Amazon S3 buckets associated with the target and attempt to list their contents."""

    # Default options
    opts = {
        "endpoints": "s3.amazonaws.com,s3-external-1.amazonaws.com,s3-us-west-1.amazonaws.com,s3-us-west-2.amazonaws.com,s3.ap-south-1.amazonaws.com,s3-ap-south-1.amazonaws.com,s3.ap-northeast-2.amazonaws.com,s3-ap-northeast-2.amazonaws.com,s3-ap-southeast-1.amazonaws.com,s3-ap-southeast-2.amazonaws.com,s3-ap-northeast-1.amazonaws.com,s3.eu-central-1.amazonaws.com,s3-eu-central-1.amazonaws.com,s3-eu-west-1.amazonaws.com,s3-sa-east-1.amazonaws.com",
        "suffixes": "test,dev,web,beta,bucket,space,files,content,data,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data",
        "_maxthreads": 20
    }

    # Option descriptions
    optdescs = {
        "endpoints": "Different S3 endpoints to check where buckets may exist, as per http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region",
        "suffixes": "List of suffixes to append to domains tried as bucket names"
    }

    results = list()
    s3results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.s3results = dict()
        self.results = list()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["DOMAIN_NAME", "LINKED_URL_EXTERNAL"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]

    def checkSite(self, url):
        res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)

        if res['code'] not in [ "301", "302", "200" ] and \
            (res['content'] is None or "NoSuchBucket" in res['content']):
            #self.sf.debug("Not a valid bucket: " + url)
            return None
        else:
            if "ListBucketResult" in res['content']:
                self.s3results[url] = res['content'].count("<Key>")
            else:
                self.s3results[url] = 0

    def threadSites(self, siteList):
        ret = list()
        self.s3results = dict()
        running = True
        i = 0
        t = []

        for site in siteList:
            if self.checkForStop():
                return None

            self.sf.info("Spawning thread to check bucket: " + site)
            t.append(threading.Thread(name='sfp_s3buckets_' + site,
                                      target=self.checkSite, args=(site,)))
            t[i].start()
            i += 1

        # Block until all threads are finished
        while running:
            found = False
            for rt in threading.enumerate():
                if rt.name.startswith("sfp_s3buckeets_"):
                    found = True

            if not found:
                running = False

            time.sleep(0.25)

        # Return once the scanning has completed
        return self.s3results

    def batchSites(self, sites):
        i = 0
        res = list()
        siteList = list()

        for site in sites:
            if i >= self.opts['_maxthreads']:
                data = self.threadSites(siteList)
                if data == None:
                    return res

                for ret in data.keys():
                    if data[ret]:
                        # bucket:filecount
                        res.append(ret + ":" + str(data[ret]))
                i = 0
                siteList = list()

            siteList.append(site)
            i += 1

        return res

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        if eventData in self.results:
            return None
        else:
            self.results.append(eventData)

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        if eventName == "LINKED_URL_EXTERNAL":
            if ".amazonaws.com" in eventData:
                b = self.sf.urlFQDN(eventData)
                evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event)
                self.notifyListeners(evt)
            return None

        targets = [ eventData.replace('.', ''), self.sf.domainKeyword(eventData, self.opts['_internettlds']) ]
        urls = list()
        for t in targets:
            for e in self.opts['endpoints'].split(','):
                suffixes = [''] + self.opts['suffixes'].split(',')
                for s in suffixes:
                    if self.checkForStop():
                        return None

                    b = t + s + "." + e
                    url = "https://" + b
                    urls.append(url)

        # Batch the scans
        ret = self.batchSites(urls)
        for b in ret:
            bucket = b.split(":")
            evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)
            self.notifyListeners(evt)
            if bucket[2] != "0":
                evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[0] + ": " + bucket[2] + " files found.",
                                      self.__name__, evt)
                self.notifyListeners(evt)

# End of sfp_s3bucket class
<commit_msg>Add bucket suffixes<commit_after># -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name:         sfp_s3bucket
# Purpose:      SpiderFoot plug-in for identifying potential S3 buckets related to
#               the target.
#
# Author:      Steve Micallef <steve@binarypool.com>
#
# Created:     24/07/2016
# Copyright:   (c) Steve Micallef 2016
# Licence:     GPL
# -------------------------------------------------------------------------------

import threading
import time
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent

class sfp_s3bucket(SpiderFootPlugin):
    """Amazon S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential Amazon S3 buckets associated with the target and attempt to list their contents."""

    # Default options
    opts = {
        "endpoints": "s3.amazonaws.com,s3-external-1.amazonaws.com,s3-us-west-1.amazonaws.com,s3-us-west-2.amazonaws.com,s3.ap-south-1.amazonaws.com,s3-ap-south-1.amazonaws.com,s3.ap-northeast-2.amazonaws.com,s3-ap-northeast-2.amazonaws.com,s3-ap-southeast-1.amazonaws.com,s3-ap-southeast-2.amazonaws.com,s3-ap-northeast-1.amazonaws.com,s3.eu-central-1.amazonaws.com,s3-eu-central-1.amazonaws.com,s3-eu-west-1.amazonaws.com,s3-sa-east-1.amazonaws.com",
        "suffixes": "test,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development",
        "_maxthreads": 20
    }

    # Option descriptions
    optdescs = {
        "endpoints": "Different S3 endpoints to check where buckets may exist, as per http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region",
        "suffixes": "List of suffixes to append to domains tried as bucket names"
    }

    results = list()
    s3results = dict()

    def setup(self, sfc, userOpts=dict()):
        self.sf = sfc
        self.s3results = dict()
        self.results = list()

        for opt in userOpts.keys():
            self.opts[opt] = userOpts[opt]

    # What events is this module interested in for input
    def watchedEvents(self):
        return ["DOMAIN_NAME", "LINKED_URL_EXTERNAL"]

    # What events this module produces
    # This is to support the end user in selecting modules based on events
    # produced.
    def producedEvents(self):
        return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]

    def checkSite(self, url):
        res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)

        if res['code'] not in [ "301", "302", "200" ] and \
            (res['content'] is None or "NoSuchBucket" in res['content']):
            #self.sf.debug("Not a valid bucket: " + url)
            return None
        else:
            if "ListBucketResult" in res['content']:
                self.s3results[url] = res['content'].count("<Key>")
            else:
                self.s3results[url] = 0

    def threadSites(self, siteList):
        ret = list()
        self.s3results = dict()
        running = True
        i = 0
        t = []

        for site in siteList:
            if self.checkForStop():
                return None

            self.sf.info("Spawning thread to check bucket: " + site)
            t.append(threading.Thread(name='sfp_s3buckets_' + site,
                                      target=self.checkSite, args=(site,)))
            t[i].start()
            i += 1

        # Block until all threads are finished
        while running:
            found = False
            for rt in threading.enumerate():
                if rt.name.startswith("sfp_s3buckeets_"):
                    found = True

            if not found:
                running = False

            time.sleep(0.25)

        # Return once the scanning has completed
        return self.s3results

    def batchSites(self, sites):
        i = 0
        res = list()
        siteList = list()

        for site in sites:
            if i >= self.opts['_maxthreads']:
                data = self.threadSites(siteList)
                if data == None:
                    return res

                for ret in data.keys():
                    if data[ret]:
                        # bucket:filecount
                        res.append(ret + ":" + str(data[ret]))
                i = 0
                siteList = list()

            siteList.append(site)
            i += 1

        return res

    # Handle events sent to this module
    def handleEvent(self, event):
        eventName = event.eventType
        srcModuleName = event.module
        eventData = event.data

        if eventData in self.results:
            return None
        else:
            self.results.append(eventData)

        self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)

        if eventName == "LINKED_URL_EXTERNAL":
            if ".amazonaws.com" in eventData:
                b = self.sf.urlFQDN(eventData)
                evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event)
                self.notifyListeners(evt)
            return None

        targets = [ eventData.replace('.', ''), self.sf.domainKeyword(eventData, self.opts['_internettlds']) ]
        urls = list()
        for t in targets:
            for e in self.opts['endpoints'].split(','):
                suffixes = [''] + self.opts['suffixes'].split(',')
                for s in suffixes:
                    if self.checkForStop():
                        return None

                    b = t + s + "." + e
                    url = "https://" + b
                    urls.append(url)

        # Batch the scans
        ret = self.batchSites(urls)
        for b in ret:
            bucket = b.split(":")
            evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)
            self.notifyListeners(evt)
            if bucket[2] != "0":
                evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[0] + ": " + bucket[2] + " files found.",
                                      self.__name__, evt)
                self.notifyListeners(evt)

# End of sfp_s3bucket class
<|endoftext|>


<commit_before>'''
Created on Dec 3, 2015

@author: kashefy
'''
import numpy as np
import h5py
import caffe

def infer_to_h5_fixed_dims(net, keys, n, dst_fpath, preserve_batch=False):
    """
    Run network inference for n batches and save results to file
    """
    dc = {k:[] for k in keys}
    for _ in range(n):
        d = forward(net, keys)
        for k in keys:
            if preserve_batch:
                dc[k].append(np.copy(d[k]))
            else:
                dc[k].extend(np.copy(d[k]))

    with h5py.File(dst_fpath, "w") as f:
        for k in keys:
            f[k] = dc[k]

    return [len(dc[k]) for k in keys]

def forward(net, keys):

    net.forward()
    return {k : net.blobs[k].data for k in keys}

if __name__ == '__main__':

    import os
    fpath_net = os.path.expanduser('~/models/dark/mnist/t0/lenet_train_test.prototxt')
    fpath_weights = os.path.expanduser('~/models/dark/mnist/t0/lenet_iter_10000.caffemodel')
    net = caffe.Net(fpath_net, fpath_weights, caffe.TEST)
    fpath = os.path.expanduser('~/models/dark/mnist/t0/x.h5')
    infer_to_h5_fixed_dims(net, ['data', 'label'], 4, fpath)

#     with h5py.File(fpath, "w") as f:
#
#         f['a'] = 0
#
#
#         f['b'] = [1, 2]
#         f['c'] = np.arange(3)
#         f['d'] = [np.array([[1,2],[4,5]], dtype=float), np.array([[1,2],[4, 5]], dtype=float)+10]

    #infer_to_h5(net, 1, ['accuracy'], fpath)

    pass<commit_msg>inference output to lmdb<commit_after>'''
Created on Dec 3, 2015

@author: kashefy
'''
import numpy as np
import h5py
import caffe
import read_lmdb
import to_lmdb

def infer_to_h5_fixed_dims(net, keys, n, dst_fpath, preserve_batch=False):
    """
    Run network inference for n batches and save results to file
    """
    dc = {k:[] for k in keys}
    for _ in range(n):
        d = forward(net, keys)
        for k in keys:
            if preserve_batch:
                dc[k].append(np.copy(d[k]))
            else:
                dc[k].extend(np.copy(d[k]))

    with h5py.File(dst_fpath, "w") as f:
        for k in keys:
            f[k] = dc[k]

    return [len(dc[k]) for k in keys]

def infer_to_lmdb(net, keys, n, dst_fpath, preserve_batch=False):
    """
    Run network inference for n batches and save results to file
    """
    dc = {k:[] for k in keys}
    for _ in range(n):
        d = forward(net, keys)
        for k in keys:
            if preserve_batch:
                dc[k].append(np.copy(d[k].astype(float)))
            else:
                dc[k].extend(np.copy(d[k].astype(float)))

    for k in keys:
        to_lmdb.arrays_to_lmdb(dc[k], dst_fpath)

    return [len(dc[k]) for k in keys]

def forward(net, keys):

    net.forward()
    return {k : net.blobs[k].data for k in keys}

if __name__ == '__main__':

    from os.path import expanduser

    fpath_net = expanduser('~/models/dark/mnist/t0/lenet_train_test_inf.prototxt')

    fpath_weights = expanduser('~/models/dark/mnist/t0/lenet_iter_10000.caffemodel')
    net = caffe.Net(fpath_net, fpath_weights, caffe.TRAIN)

    fpath = expanduser('~/models/dark/mnist/t0/mnist_dark0_sm_train')
    num_entries = read_lmdb.num_entries("/home/kashefy/data/mnist/mnist_train_lmdb")

    #infer_to_h5_fixed_dims(net, ['prob'], num_entries, "%s.h5" % fpath)
    infer_to_lmdb(net, ['prob'], num_entries, "%s_lmdb" % fpath)

    net = caffe.Net(fpath_net, fpath_weights, caffe.TEST)
    fpath = expanduser('~/models/dark/mnist/t0/mnist_dark0_sm_test')
    num_entries = read_lmdb.num_entries("/home/kashefy/data/mnist/mnist_test_lmdb")
    #infer_to_h5_fixed_dims(net, ['prob'], num_entries, "%s.h5" % fpath)
    infer_to_lmdb(net, ['prob'], num_entries, "%s_lmdb" % fpath)

    print 'done'

    #l = read_lmdb.read_labels('/home/kashefy/data/mnist/mnist_train_lmdb')
    #p = [x for x, _ in read_lmdb.read_values("~/models/dark/mnist/t0/mnist_dark0_sm_train_lmdb")]

#     with h5py.File(fpath, "w") as f:
#
#         f['a'] = 0
#
#
#         f['b'] = [1, 2]
#         f['c'] = np.arange(3)
#         f['d'] = [np.array([[1,2],[4,5]], dtype=float), np.array([[1,2],[4, 5]], dtype=float)+10]

    #infer_to_h5(net, 1, ['accuracy'], fpath)

    pass<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""
Use URL to EUPS candidate tag file to git tag repos with official version
"""

# Technical Debt
# --------------
# - completely hide eups-specifics from this file
# - support repos.yaml for github repo resolution
# - worth doing the smart thing for externals? (yes for Sims)
# - deal with authentication version

# Known Bugs
# ----------
# Yeah, the candidate logic is broken, will fix

from .. import codetools
from .. import debug, warn, error
import argparse
import codekit.pygithub as pygithub
import copy
import github
import logging
import os
import requests
import sys
import textwrap

logging.basicConfig()
logger = logging.getLogger('codekit')
eupspkg_site = 'https://eups.lsst.codes/stack/src'

class GitTagExistsError(Exception):
    pass

class RepositoryMissingTeamError(Exception):
    def __init__(self, repo, repo_teams, tag_teams):
        assert isinstance(repo, github.Repository.Repository), type(repo)

        self.repo = repo
        self.repo_teams = repo_teams
        self.tag_teams = tag_teams

    def __str__(self):
        return textwrap.dedent("""\
            No action for {repo}
              has teams: {repo_teams}
              does not belong to any of: {tag_teams}\
            """.format(
            repo=self.repo.full_name,
            repo_teams=self.repo_teams,
            tag_teams=self.tag_teams
        ))

class DogpileError(Exception):
    def __init__(self, errors, msg):
        self.errors = errors
        self.msg = msg

    def __str__(self):
        return self.msg + "\n" + "\n".join([str(e) for e in self.errors])

def parse_args():
    """Parse command-line arguments"""

    parser = argparse.ArgumentParser(
        prog='github-tag-version',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""

        Tag all repositories in a GitHub org using a team-based scheme

        Examples:
        github-tag-version --org lsst --team 'Data Management' w.2015.33 b1630

        github-tag-version --org lsst --team 'Data Management' \
            --team 'External' --candidate v11_0_rc2 11.0.rc2 b1679

        Note that the access token must have access to these oauth scopes:
            * read:org
            * repo

        The token generated by `github-auth --user` should have sufficient
        permissions.
        """),
        epilog='Part of codekit: https://github.com/lsst-sqre/sqre-codekit'
    )

    parser.add_argument('tag')
    parser.add_argument('manifest')
    parser.add_argument(
        '--org',
        required=True,
        help="Github organization")
    parser.add_argument(
        '--team',
        action='append',
        required=True,
        help="team whose repos may be tagged (can specify several times")
    parser.add_argument('--candidate')
    parser.add_argument('--dry-run', action='store_true')
    parser.add_argument(
        '--user',
        help='Name of person making the tag - defaults to gitconfig value')
    parser.add_argument(
        '--email',
        help='Email address of tagger - defaults to gitconfig value')
    parser.add_argument(
        '--token-path',
        default='~/.sq_github_token_delete',
        help='Use a token (made with github-auth) in a non-standard location')
    parser.add_argument(
        '--token',
        default=None,
        help='Literal github personal access token string')
    parser.add_argument(
        '--force-tag',
        action='store_true',
        help='Force moving pre-existing annotated git tags.')
    parser.add_argument(
        '--fail-fast',
        action='store_true',
        help='Fail immediately on github API errors.')
    parser.add_argument(
        '--no-fail-fast',
        action='store_const',
        const=False,
        dest='fail_fast',
        help='DO NOT Fail immediately on github API errors. (default)')
    parser.add_argument(
        '-d', '--debug',
        action='count',
        default=os.getenv('DM_SQUARE_DEBUG'),
        help='Debug mode')
    parser.add_argument('-v', '--version', action=codetools.ScmVersionAction)
    return parser.parse_args()

def cmp_dict(d1, d2, ignore_keys=[]):
    """Compare dicts ignoring select keys"""
    # https://stackoverflow.com/questions/10480806/compare-dictionaries-ignoring-specific-keys
    return {k: v for k, v in d1.items() if k not in ignore_keys} \
        == {k: v for k, v in d2.items() if k not in ignore_keys}

def fetch_eups_tag_file(args, eups_candidate):
    # construct url
    eupspkg_taglist = '/'.join((eupspkg_site, 'tags',
                                eups_candidate + '.list'))
    debug("fetching: {url}".format(url=eupspkg_taglist))

    r = requests.get(eupspkg_taglist)
    r.raise_for_status()

    return r.text

def parse_eups_tag_file(data):
    products = []

    for line in data.splitlines():
        if not isinstance(line, str):
            line = str(line, 'utf-8')
        # skip commented out and blank lines
        if line.startswith('#'):
            continue
        if line.startswith('EUPS'):
            continue
        if line == '':
            continue

        # extract the repo and eups tag
        (product, _, eups_version) = line.split()[0:3]

        products.append({
            'name': product,
            'eups_version': eups_version,
        })

    return products

def eups_products_to_gh_repos(
    org,
    teams,
    eupsbuild,
    eups_products,
    fail_fast=False
):
    problems = []
    gh_repos = []
    for prod in eups_products:
        debug("looking for git repo for: {prod} [{ver}]".format(
            prod=prod['name'],
            ver=prod['eups_version']
        ))

        try:
            repo = org.get_repo(prod['name'])
        except github.RateLimitExceededException:
            raise
        except github.UnknownObjectException as e:
            yikes = pygithub.CaughtUnknownObjectError(prod['name'], e)
            if fail_fast:
                raise yikes from None
            problems.append(yikes)
            error(yikes)

            continue

        debug("  found: {slug}".format(slug=repo.full_name))

        repo_teams = [t.name for t in repo.get_teams()]
        if not any(x in repo_teams for x in teams):
            yikes = RepositoryMissingTeamError(repo, repo_teams, teams)
            if fail_fast:
                raise yikes
            problems.append(yikes)
            error(yikes)

            continue

        sha = codetools.eups2git_ref(
            product=repo.name,
            eups_version=prod['eups_version'],
            build_id=eupsbuild
        )

        gh_repos.append({
            'repo': repo,
            'product': prod['name'],
            'eups_version': prod['eups_version'],
            'sha': sha,
        })

    if problems:
        msg = "{n} repo(s) have errors".format(n=len(problems))
        raise DogpileError(problems, msg)

    return gh_repos

def author_to_dict(obj):
    """Who needs a switch/case statement when you can instead use this easy to
    comprehend drivel?
    """
    def default():
        raise RuntimeError("unsupported type {t}".format(t=type(obj).__name__))

    # a more pythonic way to handle this would be several try blocks to catch
    # missing attributes
    return {
        # GitAuthor has name,email,date properties
        'GitAuthor': lambda x: {'name': x.name, 'email': x.email},
        # InputGitAuthor only has _identity, which returns a dict
        'InputGitAuthor': lambda x: x._identity,
    }.get(type(obj).__name__, lambda x: default())(obj)

def cmp_gitauthor(a, b):
    # ignore date
    if cmp_dict(author_to_dict(a), author_to_dict(b), ['date']):
        return True

    return False

def cmp_existing_git_tag(t_tag, e_tag):
    assert isinstance(t_tag, dict)
    assert isinstance(e_tag, github.GitTag.GitTag)

    # ignore date when comparing tag objects
    if t_tag['sha'] == e_tag.object.sha and \
       t_tag['message'] == e_tag.message and \
       cmp_gitauthor(t_tag['tagger'], e_tag.tagger):
        return True

    return False

def check_existing_git_tag(repo, t_tag):
    """
    Check for a pre-existng tag in the github repo.

    Parameters
    ----------
    repo : dict
        dict representing a target github repo
    t_tag: dict
        dict repesenting a target git tag

    Returns
    -------
    insync : `bool`
        True if tag exists and is in sync. False if tag does not exist.

    Raises
    ------
    GitTagExistsError
        If tag exists but is not in sync.
    """

    debug("looking for existing tag: {tag}"
          .format(tag=t_tag['name']))

    # find ref/tag by name
    e_ref = pygithub.find_tag_by_name(repo['repo'], t_tag['name'])
    if not e_ref:
        debug("  not found: {tag}".format(tag=t_tag['name']))
        return False

    # find tag object pointed to by the ref
    e_tag = repo['repo'].get_git_tag(e_ref.object.sha)
    debug("  found existing tag: {tag}".format(tag=e_tag))

    if cmp_existing_git_tag(t_tag, e_tag):
        return True

    warn(textwrap.dedent("""\
        tag {tag} already exists with conflicting values:
          existing:
            sha: {e_sha}
            message: {e_message}
            tagger: {e_tagger}
          target:
            sha: {t_sha}
            message: {t_message}
            tagger: {t_tagger}\
    """).format(
        tag=t_tag['name'],
        e_sha=e_tag['sha'],
        e_message=e_tag['message'],
        e_tagger=e_tag['tagger'],
        t_sha=t_tag['sha'],
        t_message=t_tag['message'],
        t_tagger=t_tag['tagger'],
    ))

    raise GitTagExistsError("tag {tag} alreaday exists"
                            .format(tag=e_tag))

def tag_gh_repos(
    gh_repos,
    tag_template,
    force_tag=False,
    fail_fast=False,
    dry_run=False,
):
    problems = []
    for repo in gh_repos:
        # "target tag"
        t_tag = copy.copy(tag_template)
        t_tag['sha'] = repo['sha']

        # control whether to create a new tag or update an existing one
        update_tag = False

        debug(textwrap.dedent("""\
            tagging repo: {repo}
              sha: {sha} as {gt}
              (eups version: {et})\
            """).format(
            repo=repo['repo'].full_name,
            sha=t_tag['sha'],
            gt=t_tag['name'],
            et=repo['eups_version']
        ))

        try:
            # if the existing tag is in sync, do nothing
            if check_existing_git_tag(repo, t_tag):
                warn(textwrap.dedent("""\
                    No action for {repo}
                      existing tag: {tag} is already in sync\
                    """).format(
                    repo=repo['repo'].full_name,
                    tag=t_tag['name'],
                ))

                continue
        except github.RateLimitExceededException:
            raise
        except (github.GithubException, GitTagExistsError) as e:
            yikes = pygithub.CaughtRepositoryError(repo['repo'], e)

            if force_tag:
                update_tag = True
            elif fail_fast:
                raise yikes from None
            else:
                problems.append(yikes)
                continue

        # tags are created/updated past this point
        if dry_run:
            continue

        try:
            tag_obj = repo['repo'].create_git_tag(
                t_tag['name'],
                t_tag['message'],
                t_tag['sha'],
                'commit',
                tagger=t_tag['tagger'],
            )
            debug("  created tag object {tag_obj}".format(tag_obj=tag_obj))

            if update_tag:
                ref = pygithub.find_tag_by_name(
                    repo['repo'],
                    t_tag['name'],
                    safe=False
                )
                ref.edit(tag_obj.sha, force=True)
                debug("  updated existing ref: {ref}".format(ref=ref))
            else:
                ref = repo['repo'].create_git_ref(
                    "refs/tags/{t}".format(t=t_tag['name']),
                    tag_obj.sha
                )
                debug("  created ref: {ref}".format(ref=ref))
        except github.RateLimitExceededException:
            raise
        except github.GithubException as e:
            yikes = pygithub.CaughtRepositoryError(repo['repo'], e)
            if fail_fast:
                raise yikes from None
            problems.append(yikes)
            error(yikes)

    if problems:
        msg = "{n} tag failures".format(n=len(problems))
        raise DogpileError(problems, msg)

def run():
    """Create the tag"""

    args = parse_args()

    if args.debug:
        logger.setLevel(logging.DEBUG)
    if args.debug > 1:
        github.enable_console_debug_logging()

    version = args.tag

    # if email not specified, try getting it from the gitconfig
    git_email = codetools.lookup_email(args)
    # ditto for the name of the git user
    git_user = codetools.lookup_user(args)

    # The candidate is assumed to be the requested EUPS tag unless
    # otherwise specified with the --candidate option The reason to
    # currently do this is that for weeklies and other internal builds,
    # it's okay to eups publish the weekly and git tag post-facto. However
    # for official releases, we don't want to publish until the git tag
    # goes down, because we want to eups publish the build that has the
    # official versions in the eups ref.
    candidate = args.candidate if args.candidate else args.tag

    eupsbuild = args.manifest  # sadly we need to "just" know this
    message_template = 'Version {v} release from {c}/{b}'
    message = message_template.format(v=version, c=candidate, b=eupsbuild)

    tagger = github.InputGitAuthor(
        git_user,
        git_email,
        codetools.current_timestamp()
    )
    debug(tagger)

    # all tags should be the same across repos -- just add the 'sha' key and
    # stir
    tag_template = {
        'name': version,
        'message': message,
        'tagger': tagger,
    }

    debug(tag_template)

    global g
    g = pygithub.login_github(token_path=args.token_path, token=args.token)
    org = g.get_organization(args.org)
    debug("tagging repos in github org: {org}".format(org=org.login))

    # generate eups-style version
    # eups no likey semantic versioning markup, wants underscores
    cmap = str.maketrans('.-', '__')
    eups_candidate = candidate.translate(cmap)

    manifest = fetch_eups_tag_file(args, eups_candidate)
    eups_products = parse_eups_tag_file(manifest)

    # do not fail-fast on non-write operations
    gh_repos = eups_products_to_gh_repos(
        org,
        args.team,
        eupsbuild,
        eups_products,
        fail_fast=False
    )

    tag_gh_repos(
        gh_repos,
        tag_template,
        force_tag=args.force_tag,
        fail_fast=args.fail_fast,
        dry_run=args.dry_run
    )

def main():
    try:
        run()
    except DogpileError as e:
        error(e)
        n = len(e.errors)
        sys.exit(n if n < 256 else 255)
    finally:
        if 'g' in globals():
            pygithub.debug_ratelimit(g)

if __name__ == '__main__':
    main()
<commit_msg>fix log formatting of github.GitTag objects<commit_after>#!/usr/bin/env python3
"""
Use URL to EUPS candidate tag file to git tag repos with official version
"""

# Technical Debt
# --------------
# - completely hide eups-specifics from this file
# - support repos.yaml for github repo resolution
# - worth doing the smart thing for externals? (yes for Sims)
# - deal with authentication version

# Known Bugs
# ----------
# Yeah, the candidate logic is broken, will fix

from .. import codetools
from .. import debug, warn, error
import argparse
import codekit.pygithub as pygithub
import copy
import github
import logging
import os
import requests
import sys
import textwrap

logging.basicConfig()
logger = logging.getLogger('codekit')
eupspkg_site = 'https://eups.lsst.codes/stack/src'

class GitTagExistsError(Exception):
    pass

class RepositoryMissingTeamError(Exception):
    def __init__(self, repo, repo_teams, tag_teams):
        assert isinstance(repo, github.Repository.Repository), type(repo)

        self.repo = repo
        self.repo_teams = repo_teams
        self.tag_teams = tag_teams

    def __str__(self):
        return textwrap.dedent("""\
            No action for {repo}
              has teams: {repo_teams}
              does not belong to any of: {tag_teams}\
            """.format(
            repo=self.repo.full_name,
            repo_teams=self.repo_teams,
            tag_teams=self.tag_teams
        ))

class DogpileError(Exception):
    def __init__(self, errors, msg):
        self.errors = errors
        self.msg = msg

    def __str__(self):
        return self.msg + "\n" + "\n".join([str(e) for e in self.errors])

def parse_args():
    """Parse command-line arguments"""

    parser = argparse.ArgumentParser(
        prog='github-tag-version',
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""

        Tag all repositories in a GitHub org using a team-based scheme

        Examples:
        github-tag-version --org lsst --team 'Data Management' w.2015.33 b1630

        github-tag-version --org lsst --team 'Data Management' \
            --team 'External' --candidate v11_0_rc2 11.0.rc2 b1679

        Note that the access token must have access to these oauth scopes:
            * read:org
            * repo

        The token generated by `github-auth --user` should have sufficient
        permissions.
        """),
        epilog='Part of codekit: https://github.com/lsst-sqre/sqre-codekit'
    )

    parser.add_argument('tag')
    parser.add_argument('manifest')
    parser.add_argument(
        '--org',
        required=True,
        help="Github organization")
    parser.add_argument(
        '--team',
        action='append',
        required=True,
        help="team whose repos may be tagged (can specify several times")
    parser.add_argument('--candidate')
    parser.add_argument('--dry-run', action='store_true')
    parser.add_argument(
        '--user',
        help='Name of person making the tag - defaults to gitconfig value')
    parser.add_argument(
        '--email',
        help='Email address of tagger - defaults to gitconfig value')
    parser.add_argument(
        '--token-path',
        default='~/.sq_github_token_delete',
        help='Use a token (made with github-auth) in a non-standard location')
    parser.add_argument(
        '--token',
        default=None,
        help='Literal github personal access token string')
    parser.add_argument(
        '--force-tag',
        action='store_true',
        help='Force moving pre-existing annotated git tags.')
    parser.add_argument(
        '--fail-fast',
        action='store_true',
        help='Fail immediately on github API errors.')
    parser.add_argument(
        '--no-fail-fast',
        action='store_const',
        const=False,
        dest='fail_fast',
        help='DO NOT Fail immediately on github API errors. (default)')
    parser.add_argument(
        '-d', '--debug',
        action='count',
        default=os.getenv('DM_SQUARE_DEBUG'),
        help='Debug mode')
    parser.add_argument('-v', '--version', action=codetools.ScmVersionAction)
    return parser.parse_args()

def cmp_dict(d1, d2, ignore_keys=[]):
    """Compare dicts ignoring select keys"""
    # https://stackoverflow.com/questions/10480806/compare-dictionaries-ignoring-specific-keys
    return {k: v for k, v in d1.items() if k not in ignore_keys} \
        == {k: v for k, v in d2.items() if k not in ignore_keys}

def fetch_eups_tag_file(args, eups_candidate):
    # construct url
    eupspkg_taglist = '/'.join((eupspkg_site, 'tags',
                                eups_candidate + '.list'))
    debug("fetching: {url}".format(url=eupspkg_taglist))

    r = requests.get(eupspkg_taglist)
    r.raise_for_status()

    return r.text

def parse_eups_tag_file(data):
    products = []

    for line in data.splitlines():
        if not isinstance(line, str):
            line = str(line, 'utf-8')
        # skip commented out and blank lines
        if line.startswith('#'):
            continue
        if line.startswith('EUPS'):
            continue
        if line == '':
            continue

        # extract the repo and eups tag
        (product, _, eups_version) = line.split()[0:3]

        products.append({
            'name': product,
            'eups_version': eups_version,
        })

    return products

def eups_products_to_gh_repos(
    org,
    teams,
    eupsbuild,
    eups_products,
    fail_fast=False
):
    problems = []
    gh_repos = []
    for prod in eups_products:
        debug("looking for git repo for: {prod} [{ver}]".format(
            prod=prod['name'],
            ver=prod['eups_version']
        ))

        try:
            repo = org.get_repo(prod['name'])
        except github.RateLimitExceededException:
            raise
        except github.UnknownObjectException as e:
            yikes = pygithub.CaughtUnknownObjectError(prod['name'], e)
            if fail_fast:
                raise yikes from None
            problems.append(yikes)
            error(yikes)

            continue

        debug("  found: {slug}".format(slug=repo.full_name))

        repo_teams = [t.name for t in repo.get_teams()]
        if not any(x in repo_teams for x in teams):
            yikes = RepositoryMissingTeamError(repo, repo_teams, teams)
            if fail_fast:
                raise yikes
            problems.append(yikes)
            error(yikes)

            continue

        sha = codetools.eups2git_ref(
            product=repo.name,
            eups_version=prod['eups_version'],
            build_id=eupsbuild
        )

        gh_repos.append({
            'repo': repo,
            'product': prod['name'],
            'eups_version': prod['eups_version'],
            'sha': sha,
        })

    if problems:
        msg = "{n} repo(s) have errors".format(n=len(problems))
        raise DogpileError(problems, msg)

    return gh_repos

def author_to_dict(obj):
    """Who needs a switch/case statement when you can instead use this easy to
    comprehend drivel?
    """
    def default():
        raise RuntimeError("unsupported type {t}".format(t=type(obj).__name__))

    # a more pythonic way to handle this would be several try blocks to catch
    # missing attributes
    return {
        # GitAuthor has name,email,date properties
        'GitAuthor': lambda x: {'name': x.name, 'email': x.email},
        # InputGitAuthor only has _identity, which returns a dict
        'InputGitAuthor': lambda x: x._identity,
    }.get(type(obj).__name__, lambda x: default())(obj)

def cmp_gitauthor(a, b):
    # ignore date
    if cmp_dict(author_to_dict(a), author_to_dict(b), ['date']):
        return True

    return False

def cmp_existing_git_tag(t_tag, e_tag):
    assert isinstance(t_tag, dict)
    assert isinstance(e_tag, github.GitTag.GitTag)

    # ignore date when comparing tag objects
    if t_tag['sha'] == e_tag.object.sha and \
       t_tag['message'] == e_tag.message and \
       cmp_gitauthor(t_tag['tagger'], e_tag.tagger):
        return True

    return False

def check_existing_git_tag(repo, t_tag):
    """
    Check for a pre-existng tag in the github repo.

    Parameters
    ----------
    repo : dict
        dict representing a target github repo
    t_tag: dict
        dict repesenting a target git tag

    Returns
    -------
    insync : `bool`
        True if tag exists and is in sync. False if tag does not exist.

    Raises
    ------
    GitTagExistsError
        If tag exists but is not in sync.
    """

    debug("looking for existing tag: {tag}"
          .format(tag=t_tag['name']))

    # find ref/tag by name
    e_ref = pygithub.find_tag_by_name(repo['repo'], t_tag['name'])
    if not e_ref:
        debug("  not found: {tag}".format(tag=t_tag['name']))
        return False

    # find tag object pointed to by the ref
    e_tag = repo['repo'].get_git_tag(e_ref.object.sha)
    debug("  found existing tag: {tag}".format(tag=e_tag))

    if cmp_existing_git_tag(t_tag, e_tag):
        return True

    warn(textwrap.dedent("""\
        tag {tag} already exists with conflicting values:
          existing:
            sha: {e_sha}
            message: {e_message}
            tagger: {e_tagger}
          target:
            sha: {t_sha}
            message: {t_message}
            tagger: {t_tagger}\
    """).format(
        tag=t_tag['name'],
        e_sha=e_tag.sha,
        e_message=e_tag.message,
        e_tagger=e_tag.tagger,
        t_sha=t_tag['sha'],
        t_message=t_tag['message'],
        t_tagger=t_tag['tagger'],
    ))

    raise GitTagExistsError("tag {tag} alreaday exists"
                            .format(tag=e_tag))

def tag_gh_repos(
    gh_repos,
    tag_template,
    force_tag=False,
    fail_fast=False,
    dry_run=False,
):
    problems = []
    for repo in gh_repos:
        # "target tag"
        t_tag = copy.copy(tag_template)
        t_tag['sha'] = repo['sha']

        # control whether to create a new tag or update an existing one
        update_tag = False

        debug(textwrap.dedent("""\
            tagging repo: {repo}
              sha: {sha} as {gt}
              (eups version: {et})\
            """).format(
            repo=repo['repo'].full_name,
            sha=t_tag['sha'],
            gt=t_tag['name'],
            et=repo['eups_version']
        ))

        try:
            # if the existing tag is in sync, do nothing
            if check_existing_git_tag(repo, t_tag):
                warn(textwrap.dedent("""\
                    No action for {repo}
                      existing tag: {tag} is already in sync\
                    """).format(
                    repo=repo['repo'].full_name,
                    tag=t_tag['name'],
                ))

                continue
        except github.RateLimitExceededException:
            raise
        except (github.GithubException, GitTagExistsError) as e:
            yikes = pygithub.CaughtRepositoryError(repo['repo'], e)

            if force_tag:
                update_tag = True
            elif fail_fast:
                raise yikes from None
            else:
                problems.append(yikes)
                continue

        # tags are created/updated past this point
        if dry_run:
            continue

        try:
            tag_obj = repo['repo'].create_git_tag(
                t_tag['name'],
                t_tag['message'],
                t_tag['sha'],
                'commit',
                tagger=t_tag['tagger'],
            )
            debug("  created tag object {tag_obj}".format(tag_obj=tag_obj))

            if update_tag:
                ref = pygithub.find_tag_by_name(
                    repo['repo'],
                    t_tag['name'],
                    safe=False
                )
                ref.edit(tag_obj.sha, force=True)
                debug("  updated existing ref: {ref}".format(ref=ref))
            else:
                ref = repo['repo'].create_git_ref(
                    "refs/tags/{t}".format(t=t_tag['name']),
                    tag_obj.sha
                )
                debug("  created ref: {ref}".format(ref=ref))
        except github.RateLimitExceededException:
            raise
        except github.GithubException as e:
            yikes = pygithub.CaughtRepositoryError(repo['repo'], e)
            if fail_fast:
                raise yikes from None
            problems.append(yikes)
            error(yikes)

    if problems:
        msg = "{n} tag failures".format(n=len(problems))
        raise DogpileError(problems, msg)

def run():
    """Create the tag"""

    args = parse_args()

    if args.debug:
        logger.setLevel(logging.DEBUG)
    if args.debug > 1:
        github.enable_console_debug_logging()

    version = args.tag

    # if email not specified, try getting it from the gitconfig
    git_email = codetools.lookup_email(args)
    # ditto for the name of the git user
    git_user = codetools.lookup_user(args)

    # The candidate is assumed to be the requested EUPS tag unless
    # otherwise specified with the --candidate option The reason to
    # currently do this is that for weeklies and other internal builds,
    # it's okay to eups publish the weekly and git tag post-facto. However
    # for official releases, we don't want to publish until the git tag
    # goes down, because we want to eups publish the build that has the
    # official versions in the eups ref.
    candidate = args.candidate if args.candidate else args.tag

    eupsbuild = args.manifest  # sadly we need to "just" know this
    message_template = 'Version {v} release from {c}/{b}'
    message = message_template.format(v=version, c=candidate, b=eupsbuild)

    tagger = github.InputGitAuthor(
        git_user,
        git_email,
        codetools.current_timestamp()
    )
    debug(tagger)

    # all tags should be the same across repos -- just add the 'sha' key and
    # stir
    tag_template = {
        'name': version,
        'message': message,
        'tagger': tagger,
    }

    debug(tag_template)

    global g
    g = pygithub.login_github(token_path=args.token_path, token=args.token)
    org = g.get_organization(args.org)
    debug("tagging repos in github org: {org}".format(org=org.login))

    # generate eups-style version
    # eups no likey semantic versioning markup, wants underscores
    cmap = str.maketrans('.-', '__')
    eups_candidate = candidate.translate(cmap)

    manifest = fetch_eups_tag_file(args, eups_candidate)
    eups_products = parse_eups_tag_file(manifest)

    # do not fail-fast on non-write operations
    gh_repos = eups_products_to_gh_repos(
        org,
        args.team,
        eupsbuild,
        eups_products,
        fail_fast=False
    )

    tag_gh_repos(
        gh_repos,
        tag_template,
        force_tag=args.force_tag,
        fail_fast=args.fail_fast,
        dry_run=args.dry_run
    )

def main():
    try:
        run()
    except DogpileError as e:
        error(e)
        n = len(e.errors)
        sys.exit(n if n < 256 else 255)
    finally:
        if 'g' in globals():
            pygithub.debug_ratelimit(g)

if __name__ == '__main__':
    main()
<|endoftext|>