<commit_before># -*- encoding: utf-8 -*-
# Author: Epix
import os
from urllib.parse import quote, urljoin

from lxml import etree
from requests_futures.sessions import FuturesSession

from Cache import *
from account import account
from settings import PROXIES, HEADERS, CACHE_DIR

class Pixiv:
    cookies_filename = os.path.join(os.path.dirname(__file__), 'cookies.pkl')
    base_urls = {
        'following_illust': 'http://www.pixiv.net/bookmark_new_illust.php?p=%s',
        'login': 'https://www.secure.pixiv.net/login.php',
        'test_login': 'http://www.pixiv.net/stacc/?mode=unify',
        'home': 'http://www.pixiv.net/',
    }
    # session = requests.Session()
    session = FuturesSession(max_workers=10)
    session.proxies = PROXIES
    session.headers = HEADERS
    session.verify = False
    is_login = False
    cache_db = CacheDb('cache_db.pkl')

    def __init__(self):
        try:
            self.session.cookies = pickle.load(open(self.cookies_filename, 'rb'))
        except IOError:
            pass

    def login(self):
        self.test_login()
        if self.is_login is False:
            self.do_login()
        print(self.is_login)

    def test_login(self):
        response = self.get(url=self.base_urls['test_login'], allow_redirects=False)
        if response.status_code == 200:
            self.is_login = True

    def do_login(self):
        account_extra = ('mode', 'login'), ('skip', '1')
        response = self.post(url=self.base_urls['login'], data=dict(account_extra, **account))
        if response.url == 'http://www.pixiv.net/':
            self.is_login = True
            pickle.dump(self.session.cookies, open(self.cookies_filename, 'wb'))

    def get(self, url, **kwargs):
        return self.session.get(url=url, **kwargs).result()

    def post(self, url, data=None, **kwargs):
        return self.session.post(url=url, data=data, **kwargs).result()

    def get_following(self, page_number):
        response = self.get(self.base_urls['following_illust'] % page_number)
        result = list(self.parse_list(response))
        return result

    def parse_list(self, response):
        tree = etree.HTML(response.text)
        illusts = tree.xpath('//li[@class="image-item "]')
        for illust in illusts:
            illust_url = illust.xpath('./a/@href')[0]
            illust_id = int(illust_url.split('=')[-1])
            cache_data = self.cache_db.get(illust_id)
            if cache_data:
                yield cache_data
            else:
                illust_author = illust.xpath('./a[@class="user ui-profile-popup"]')[0]
                illust_author_name = illust_author.xpath('./@data-user_name')[0]
                illust_author_id = illust_author.xpath('./@data-user_id')[0]
                illust_title = illust.xpath('./a/h1[@class="title"]/text()')[0]
                illust_thumbnail_url = illust.xpath('./a/div/img[@class="_thumbnail"]/@src')[0]
                self.cache_pic(illust_thumbnail_url, response.url)
                class_names = illust.xpath('./a[1]/@class')[0]
                is_multi = 'multiple' in class_names
                is_ugoku = 'ugoku-illust' in class_names
                illust_special = ''
                if is_multi:
                    illust_url = illust_url.replace('medium', 'manga')
                if is_multi:
                    illust_special = 'multi'
                elif is_ugoku:
                    illust_special = 'ugoku'
                illust_response = self.get(urljoin(self.base_urls['home'], illust_url),
                                           headers={'Referer': response.url})
                meta = {'url': illust_url,
                        'title': illust_title,
                        'author_name': illust_author_name,
                        'author_id': illust_author_id,
                        'thumbnail': self.quote_url(illust_thumbnail_url),
                        'special': illust_special}
                if is_multi:
                    illust_data = self.parse_manga(illust_response, meta)
                elif is_ugoku:
                    illust_data = self.parse_ugoku(illust_response, meta)
                else:
                    illust_data = self.parse_illust(illust_response, meta)
                self.cache_db.set(illust_id, illust_data)
                yield illust_data

    def parse_illust(self, response, meta):
        return self.parse_by_xpath(response, meta, '//div[starts-with(@class,"_illust_modal")]/div[@class="wrapper"]')

    def parse_manga(self, response, meta):
        return self.parse_by_xpath(response, meta, '//div[@class="item-container"]')

    def parse_ugoku(self, response, meta):
        d = dict(meta)
        d['img'].append('')
        return d

    def parse_by_xpath(self, response, meta, xpath_str):
        d = dict(meta)
        d['img'] = []
        tree = etree.HTML(response.text)
        pics = tree.xpath(xpath_str)
        for pic in pics:
            img_src = pic.xpath('./img/@data-src')[0]
            self.cache_pic(img_src, response.url)
            d['img'].append(self.quote_url(img_src))
        return d

    def cache_pic(self, url, referer):
        file_name = self.quote_url(url)
        file_path = os.path.join(CACHE_DIR, file_name)
        if os.path.isfile(file_path):
            pass
        else:
            self.session.get(url, headers={'Referer': referer}, background_callback=self.save_pic)

    def quote_url(self, url):
        return quote(url, safe='')

    def save_pic(self, session, response):
        url = response.url
        file_name = self.quote_url(url)
        file_path = os.path.join(CACHE_DIR, file_name)
        with open(file_path, 'wb') as f:
            f.write(response.content)
<commit_msg>auto retry<commit_after># -*- encoding: utf-8 -*-
# Author: Epix
import os
from urllib.parse import quote, urljoin

from lxml import etree
from requests_futures.sessions import FuturesSession

from Cache import *
from account import account
from settings import PROXIES, HEADERS, CACHE_DIR

class Pixiv:
    cookies_filename = os.path.join(os.path.dirname(__file__), 'cookies.pkl')
    base_urls = {
        'following_illust': 'http://www.pixiv.net/bookmark_new_illust.php?p=%s',
        'login': 'https://www.secure.pixiv.net/login.php',
        'test_login': 'http://www.pixiv.net/stacc/?mode=unify',
        'home': 'http://www.pixiv.net/',
    }
    # session = requests.Session()
    session = FuturesSession(max_workers=10)
    session.proxies = PROXIES
    session.headers = HEADERS
    session.verify = False
    is_login = False
    cache_db = CacheDb('cache_db.pkl')

    def __init__(self):
        try:
            self.session.cookies = pickle.load(open(self.cookies_filename, 'rb'))
        except IOError:
            pass

    def login(self):
        self.test_login()
        if self.is_login is False:
            self.do_login()
        print(self.is_login)

    def test_login(self):
        response = self.get(url=self.base_urls['test_login'], allow_redirects=False)
        if response.status_code == 200:
            self.is_login = True

    def do_login(self):
        account_extra = ('mode', 'login'), ('skip', '1')
        response = self.post(url=self.base_urls['login'], data=dict(account_extra, **account))
        if response.url == 'http://www.pixiv.net/':
            self.is_login = True
            pickle.dump(self.session.cookies, open(self.cookies_filename, 'wb'))

    def get(self, url, **kwargs):
        r = self.session.get(url=url, **kwargs).result()
        if r.status_code in (200, 301, 302, 500):
            return r
        else:
            return self.get(url, **kwargs)

    def post(self, url, data=None, **kwargs):
        return self.session.post(url=url, data=data, **kwargs).result()

    def get_following(self, page_number):
        response = self.get(self.base_urls['following_illust'] % page_number)
        result = list(self.parse_list(response))
        return result

    def parse_list(self, response):
        tree = etree.HTML(response.text)
        illusts = tree.xpath('//li[@class="image-item "]')
        for illust in illusts:
            illust_url = illust.xpath('./a/@href')[0]
            illust_id = int(illust_url.split('=')[-1])
            cache_data = self.cache_db.get(illust_id)
            if cache_data:
                yield cache_data
            else:
                illust_author = illust.xpath('./a[@class="user ui-profile-popup"]')[0]
                illust_author_name = illust_author.xpath('./@data-user_name')[0]
                illust_author_id = illust_author.xpath('./@data-user_id')[0]
                illust_title = illust.xpath('./a/h1[@class="title"]/text()')[0]
                illust_thumbnail_url = illust.xpath('./a/div/img[@class="_thumbnail"]/@src')[0]
                self.cache_pic(illust_thumbnail_url, response.url)
                class_names = illust.xpath('./a[1]/@class')[0]
                is_multi = 'multiple' in class_names
                is_ugoku = 'ugoku-illust' in class_names
                illust_special = ''
                if is_multi:
                    illust_url = illust_url.replace('medium', 'manga')
                if is_multi:
                    illust_special = 'multi'
                elif is_ugoku:
                    illust_special = 'ugoku'
                illust_response = self.get(urljoin(self.base_urls['home'], illust_url),
                                           headers={'Referer': response.url})
                meta = {'url': illust_url,
                        'title': illust_title,
                        'author_name': illust_author_name,
                        'author_id': illust_author_id,
                        'thumbnail': self.quote_url(illust_thumbnail_url),
                        'special': illust_special}
                if is_multi:
                    illust_data = self.parse_manga(illust_response, meta)
                elif is_ugoku:
                    illust_data = self.parse_ugoku(illust_response, meta)
                else:
                    illust_data = self.parse_illust(illust_response, meta)
                self.cache_db.set(illust_id, illust_data)
                yield illust_data

    def parse_illust(self, response, meta):
        return self.parse_by_xpath(response, meta, '//div[starts-with(@class,"_illust_modal")]/div[@class="wrapper"]')

    def parse_manga(self, response, meta):
        return self.parse_by_xpath(response, meta, '//div[@class="item-container"]')

    def parse_ugoku(self, response, meta):
        d = dict(meta)
        d['img'] = []
        d['img'].append('')
        return d

    def parse_by_xpath(self, response, meta, xpath_str):
        d = dict(meta)
        d['img'] = []
        tree = etree.HTML(response.text)
        pics = tree.xpath(xpath_str)
        for pic in pics:
            img_src = pic.xpath('./img/@data-src')[0]
            self.cache_pic(img_src, response.url)
            d['img'].append(self.quote_url(img_src))
        return d

    def cache_pic(self, url, referer):
        file_name = self.quote_url(url)
        file_path = os.path.join(CACHE_DIR, file_name)
        if os.path.isfile(file_path):
            pass
        else:
            self.session.get(url, headers={'Referer': referer}, background_callback=self.save_pic)

    def quote_url(self, url):
        return quote(url, safe='')

    def save_pic(self, session, response):
        if response.status_code in (200, 301, 302, 500):
            url = response.url
            file_name = self.quote_url(url)
            file_path = os.path.join(CACHE_DIR, file_name)
            with open(file_path, 'wb') as f:
                f.write(response.content)
        else:
            self.session.get(response.url, headers=response.request.headers, background_callback=self.save_pic)
<|endoftext|>


<commit_before>'''
Senders are callables which call subsiquent methods when called
Receivers are callables which receive calls from Senders
Messengers provide the functionality of both Senders and Receivers

The call function formats whatever you pass it, for the internals of a
receiver. If you don't return a value, then any receivers connected to the
sender will not be called.

Any receivers attached to a sender must be callable using the same signature
as the signature in your return call() statement.

Senders and receivers can then be connected together using the >> syntax, and
disconnected using // syntax.

@sender
def my_sender(i):
	print i
	return call(i, 1)  #return a value for a receiver

@messenger
def my_messenger(i, v):
	return call(i + v)

@receiver
def my_receiver(i):
	print i

my_sender >> my_messenger >> my_receiver
my_sender(1)
my_sender // my_messenger // my_receiver
'''

from functools import update_wrapper
from importlib import import_module

def call(*args, **kwargs):
    return args, kwargs

class wrapper(object):
    def __init__(self, func, inst=None, type=None):
        update_wrapper(self, func)
        self.inst = inst
        self.func = func.__get__(inst, type) if inst else func

    def __call__(self, *args, **kwargs):
        return self.func(*args, **kwargs)

    def __get__(self, inst, type):
        if not inst or self.__class__ is inst.__class__: return self
        context = self.__class__(self.func, inst, type)
        setattr(inst, self.__name__, context)
        return context

    def __getstate__(self):
        return (self.func.im_self if self.inst else self.func.__module__,
                self.__name__,
                self.senders if hasattr(self, 'senders') else None,
                self.receivers if hasattr(self,'receivers') else None)

    def __setstate__(self, state):
        inst, name, senders, receivers = state
        self.inst = import_module(inst) if type(inst) is str else inst
        self.func = getattr(self.inst, name).func
        if senders is not None: self.senders = senders
        if receivers is not None: self.receivers = receivers

class sender(wrapper):
    def __init__(self, *args):
        wrapper.__init__(self, *args)
        self.receivers = set()

    def __call__(self, *args, **kwargs):
        call = self.func(*args, **kwargs)
        if call is None: return
        args, kwargs = call
        for receiver in self.receivers:
            receiver(*args, **kwargs)

    def __rshift__(self, target):
        self.receivers.add(target)
        target.senders.add(self)
        return target

    def __floordiv__(self, target):
        self.receivers.remove(target)
        target.senders.remove(self)
        return target

class receiver(wrapper):
    def __init__(self, *args):
        wrapper.__init__(self, *args)
        self.senders = set()

class messenger(sender, receiver):
    def __init__(self, *args):
        sender.__init__(self, *args)
        receiver.__init__(self, *args)
<commit_msg>Put method binding into seperate class<commit_after>'''
Senders are callables which call subsiquent methods when called
Receivers are callables which receive calls from Senders
Messengers provide the functionality of both Senders and Receivers

The call function formats whatever you pass it, for the internals of a
receiver. If you don't return a value, then any receivers connected to the
sender will not be called.

Any receivers attached to a sender must be callable using the same signature
as the signature in your return call() statement.

Senders and receivers can then be connected together using the >> syntax, and
disconnected using // syntax.

@sender
def my_sender(i):
	print i
	return call(i, 1)  #return a value for a receiver

@messenger
def my_messenger(i, v):
	return call(i + v)

@receiver
def my_receiver(i):
	print i

my_sender >> my_messenger >> my_receiver
my_sender(1)
my_sender // my_messenger // my_receiver
'''

from functools import update_wrapper
from importlib import import_module

def call(*args, **kwargs):
    return args, kwargs

class binding(object):
    def __init__(self, func, inst, type):
        self.func = func.__get__(inst, type) if inst else func

    def __getstate__(self):
        return (self.func.__self__ if hasattr(self.func, '__self__') else self.func.__module__,
                self.func.__name__)

    def __setstate__(self, state):
        inst, name = state
        inst = import_module(inst) if type(inst) is str else inst
        self.func = getattr(inst, name).binding.func

class wrapper(object):
    def __init__(self, func, inst=None, type=None):
        update_wrapper(self, func)
        self.binding = binding(func, inst, type)

    def __call__(self, *args, **kwargs):
        return self.binding.func(*args, **kwargs)

    def __get__(self, inst, type):
        if not inst or self.__class__ is inst.__class__: return self
        wrapper = self.__class__(self.binding.func, inst, type)
        setattr(inst, self.__name__, wrapper)
        return wrapper

class sender(wrapper):
    def __init__(self, *args):
        wrapper.__init__(self, *args)
        self.receivers = set()

    def __call__(self, *args, **kwargs):
        call = self.binding.func(*args, **kwargs)
        if call is None: return
        args, kwargs = call
        for receiver in self.receivers:
            receiver(*args, **kwargs)

    def __rshift__(self, target):
        self.receivers.add(target)
        target.senders.add(self)
        return target

    def __floordiv__(self, target):
        self.receivers.remove(target)
        target.senders.remove(self)
        return target

class receiver(wrapper):
    def __init__(self, *args):
        wrapper.__init__(self, *args)
        self.senders = set()

class messenger(sender, receiver):
    def __init__(self, *args):
        sender.__init__(self, *args)
        receiver.__init__(self, *args)
<|endoftext|>


<commit_before>'''
Senders are callables which call subsiquent methods when called
Receivers are callables which receive calls from Senders
Messengers provide the functionality of both Senders and Receivers

The call function formats whatever you pass it, for the internals of a
receiver. If you don't return a value, then any receivers connected to the
sender will not be called.

Any receivers attached to a sender must be callable using the same signature
as the signature in your return call() statement.

Senders and receivers can then be connected together using the >> syntax, and
disconnected using // syntax.

@sender
def my_sender(i):
	print i
	return call(i, 1)  #return a value for a receiver

@messenger
def my_messenger(i, v):
	return call(i + v)

@receiver
def my_receiver(i):
	print i

my_sender >> my_messenger >> my_receiver
my_sender(1)
my_sender // my_messenger // my_receiver
'''

from functools import update_wrapper

def call(*args, **kwargs):
    return args, kwargs

class wrapper(object):
    def __init__(self, func, inst=None, type=None):
        update_wrapper(self, func)
        self.func = func.__get__(inst, type) if inst else func

    def __call__(self, *args, **kwargs):
        return self.func(*args, **kwargs)

    def __get__(self, inst, type):
        if self.inst or not inst: return self
        context = self.__class__(self.func, inst, type)
        setattr(inst, self.__name__, context)
        return context

class sender(wrapper):
    def __init__(self, *args):
        wrapper.__init__(self, *args)
        self.receivers = set()

    def __call__(self, *args, **kwargs):
        call = self.func(*args, **kwargs)
        if call is None: return
        args, kwargs = call
        for receiver in self.receivers:
            receiver(*args, **kwargs)

    def __rshift__(self, target):
        self.receivers.add(target)
        target.senders.add(self)
        return target

    def __floordiv__(self, target):
        self.receivers.remove(target)
        target.senders.remove(self)
        return target

class receiver(wrapper):
    def __init__(self, *args):
        wrapper.__init__(self, *args)
        self.senders = set()

class messenger(sender, receiver):
    def __init__(self, *args):
        sender.__init__(self, *args)
        receiver.__init__(self, *args)
<commit_msg>Removed storing of inst and type values<commit_after>'''
Senders are callables which call subsiquent methods when called
Receivers are callables which receive calls from Senders
Messengers provide the functionality of both Senders and Receivers

The call function formats whatever you pass it, for the internals of a
receiver. If you don't return a value, then any receivers connected to the
sender will not be called.

Any receivers attached to a sender must be callable using the same signature
as the signature in your return call() statement.

Senders and receivers can then be connected together using the >> syntax, and
disconnected using // syntax.

@sender
def my_sender(i):
	print i
	return call(i, 1)  #return a value for a receiver

@messenger
def my_messenger(i, v):
	return call(i + v)

@receiver
def my_receiver(i):
	print i

my_sender >> my_messenger >> my_receiver
my_sender(1)
my_sender // my_messenger // my_receiver
'''

from functools import update_wrapper

def call(*args, **kwargs):
    return args, kwargs

class wrapper(object):
    def __init__(self, func, inst=None, type=None):
        update_wrapper(self, func)
        self.func = func.__get__(inst, type) if inst else func

    def __call__(self, *args, **kwargs):
        return self.func(*args, **kwargs)

    def __get__(self, inst, type):
        if not inst or self.__class__ is inst.__class__: return self
        context = self.__class__(self.func, inst, type)
        setattr(inst, self.__name__, context)
        return context

class sender(wrapper):
    def __init__(self, *args):
        wrapper.__init__(self, *args)
        self.receivers = set()

    def __call__(self, *args, **kwargs):
        call = self.func(*args, **kwargs)
        if call is None: return
        args, kwargs = call
        for receiver in self.receivers:
            receiver(*args, **kwargs)

    def __rshift__(self, target):
        self.receivers.add(target)
        target.senders.add(self)
        return target

    def __floordiv__(self, target):
        self.receivers.remove(target)
        target.senders.remove(self)
        return target

class receiver(wrapper):
    def __init__(self, *args):
        wrapper.__init__(self, *args)
        self.senders = set()

class messenger(sender, receiver):
    def __init__(self, *args):
        sender.__init__(self, *args)
        receiver.__init__(self, *args)

class T:
    @sender
    def f(self, i): return call(i)
    @receiver
    def b(self, i): print i
<|endoftext|>


<commit_before># Copyright (C) 2009, Thomas Leonard
# See the README file for details, or visit http://0install.net.

import gtk
from dialog import Template
from zeroinstall.gtkui import help_box
from zeroinstall.injector.model import network_levels
from zeroinstall.injector import trust, gpg
from freshness import freshness_levels, Freshness
from sets import Set

tips = gtk.Tooltips()

SHOW_CACHE = 0

network_names = {
	'off-line': _('Off-line'),
	'minimal': _('Minimal'),
	'full': _('Full'),
}

class Preferences:
	def __init__(self, policy):
		widgets = Template('preferences_box')

		self.window = widgets.get_widget('preferences_box')
		self.window.connect('destroy', lambda w: self.destroyed())

		network = widgets.get_widget('network_use')
		for level in network_levels:
			network.append_text(network_names.get(level, level.capitalize()))
		network.set_active(list(network_levels).index(policy.network_use))

		def set_network_use(combo):
			policy.network_use = network_levels[network.get_active()]
			policy.save_config()
			policy.solve_with_downloads()
		network.connect('changed', set_network_use)

		# Freshness
		times = [x.time for x in freshness_levels]
		if policy.freshness not in times:
			freshness_levels.append(Freshness(policy.freshness,
							  '%d seconds' % policy.freshness))
			times.append(policy.freshness)
		eb = gtk.EventBox()	# For the tooltip
		freshness = widgets.get_widget('freshness')
		for level in freshness_levels:
			freshness.append_text(str(level))
		freshness.set_active(times.index(policy.freshness))
		def set_freshness(combo):
			policy.freshness = freshness_levels[freshness.get_active()].time
			policy.save_config()
			policy.recalculate()
		freshness.connect('changed', set_freshness)

		stable_toggle = widgets.get_widget('help_test')
		stable_toggle.set_active(policy.help_with_testing)
		def toggle_stability(toggle):
			policy.help_with_testing = toggle.get_active()
			policy.save_config()
			policy.recalculate()
		stable_toggle.connect('toggled', toggle_stability)

		# Keys
		keys_view = widgets.get_widget('trusted_keys')
		KeyList(keys_view)

		# Responses
		self.window.set_default_response(gtk.RESPONSE_CLOSE)
		self.window.default_widget.grab_focus()

		def response(dialog, resp):
			if resp in (gtk.RESPONSE_CLOSE, gtk.RESPONSE_DELETE_EVENT):
				self.window.destroy()
			elif resp == gtk.RESPONSE_HELP:
				gui_help.display()
		self.window.connect('response', response)

		self.window.set_default_size(-1, gtk.gdk.screen_height() / 3)

	def destroyed(self):
		global preferences_box
		preferences_box = None

class KeyList:
	def __init__(self, tv):
		self.trusted_keys = gtk.TreeStore(str, object)
		tv.set_model(self.trusted_keys)
		tc = gtk.TreeViewColumn(_('Trusted keys'), gtk.CellRendererText(), text = 0)
		tv.append_column(tc)
		trust.trust_db.ensure_uptodate()

		def update_keys():
			# Remember which ones are open
			expanded_elements = set()
			def add_row(tv, path):
				if len(path) == 1:
					domain = self.trusted_keys[path][0]
					expanded_elements.add(domain)
			tv.map_expanded_rows(add_row)

			self.trusted_keys.clear()
			domains = {}

			keys = gpg.load_keys(trust.trust_db.keys.keys())

			for fingerprint in keys:
				for domain in trust.trust_db.keys[fingerprint]:
					if domain not in domains:
						domains[domain] = Set()
					domains[domain].add(keys[fingerprint])
			for domain in domains:
				iter = self.trusted_keys.append(None, [domain, None])
				for key in domains[domain]:
					self.trusted_keys.append(iter, [key.name, key])

			def may_expand(model, path, iter):
				if len(path) == 1:
					if model[iter][0] in expanded_elements:
						tv.expand_row(path, False)
			self.trusted_keys.foreach(may_expand)

		trust.trust_db.watchers.append(update_keys)
		tv.connect('destroy', lambda w: trust.trust_db.watchers.remove(update_keys))

		update_keys()

		def remove_key(fingerprint, domain):
			trust.trust_db.untrust_key(fingerprint, domain)
			trust.trust_db.notify()

		def trusted_keys_button_press(tv, bev):
			if bev.type == gtk.gdk.BUTTON_PRESS and bev.button == 3:
				pos = tv.get_path_at_pos(int(bev.x), int(bev.y))
				if not pos:
					return False
				path, col, x, y = pos
				if len(path) != 2:
					return False

				domain = self.trusted_keys[path[:-1]][0]
				key = self.trusted_keys[path][1]

				menu = gtk.Menu()

				item = gtk.MenuItem(_('Remove key for "%s"') % key.get_short_name())
				item.connect('activate',
					lambda item, fp = key.fingerprint, d = domain: remove_key(fp, d))
				item.show()
				menu.append(item)

				menu.popup(None, None, None, bev.button, bev.time)
				return True
			return False
		tv.connect('button-press-event', trusted_keys_button_press)

preferences_box = None
def show_preferences(policy):
	global preferences_box
	if preferences_box:
		preferences_box.destroy()
	preferences_box = Preferences(policy)
	preferences_box.window.show()

gui_help = help_box.HelpBox("Zero Install Preferences Help",
('Overview', """

There are three ways to control which implementations are chosen. You can adjust the \
network policy and the overall stability policy, which affect all interfaces, or you \
can edit the policy of individual interfaces."""),

('Network use', """
The 'Network use' option controls how the injector uses the network. If off-line, \
the network is not used at all. If 'Minimal' is selected then the injector will use \
the network if needed, but only if it has no choice. It will run an out-of-date \
version rather than download a newer one. If 'Full' is selected, the injector won't \
worry about how much it downloads, but will always pick the version it thinks is best."""),

('Freshness', """
The feed files, which provide the information about which versions are \
available, are also cached. To update them, click on 'Refresh all now'. You can also \
get the injector to check for new versions automatically from time to time using \
the Freshness setting."""),

('Help test new versions', """
The overall stability policy can either be to prefer stable versions, or to help test \
new versions. Choose whichever suits you. Since different programmers have different \
ideas of what 'stable' means, you may wish to override this on a per-interface basis.

To set the policy for an interface individually, select it in the main window and \
click on 'Interface Properties'. See that dialog's help text for more information."""),

('Security', """
This section lists all keys which you currently trust. When fetching a new program or \
updates for an existing one, the feed must be signed by one of these keys. If not, \
you will be prompted to confirm that you trust the new key, and it will then be added \
to this list. To remove a key, right-click on it and choose 'Remove' from the menu."""),
)
<commit_msg>Fix Python 2.6 deprecation warning<commit_after># Copyright (C) 2009, Thomas Leonard
# See the README file for details, or visit http://0install.net.

import gtk
from dialog import Template
from zeroinstall.gtkui import help_box
from zeroinstall.injector.model import network_levels
from zeroinstall.injector import trust, gpg
from freshness import freshness_levels, Freshness

tips = gtk.Tooltips()

SHOW_CACHE = 0

network_names = {
	'off-line': _('Off-line'),
	'minimal': _('Minimal'),
	'full': _('Full'),
}

class Preferences:
	def __init__(self, policy):
		widgets = Template('preferences_box')

		self.window = widgets.get_widget('preferences_box')
		self.window.connect('destroy', lambda w: self.destroyed())

		network = widgets.get_widget('network_use')
		for level in network_levels:
			network.append_text(network_names.get(level, level.capitalize()))
		network.set_active(list(network_levels).index(policy.network_use))

		def set_network_use(combo):
			policy.network_use = network_levels[network.get_active()]
			policy.save_config()
			policy.solve_with_downloads()
		network.connect('changed', set_network_use)

		# Freshness
		times = [x.time for x in freshness_levels]
		if policy.freshness not in times:
			freshness_levels.append(Freshness(policy.freshness,
							  '%d seconds' % policy.freshness))
			times.append(policy.freshness)
		eb = gtk.EventBox()	# For the tooltip
		freshness = widgets.get_widget('freshness')
		for level in freshness_levels:
			freshness.append_text(str(level))
		freshness.set_active(times.index(policy.freshness))
		def set_freshness(combo):
			policy.freshness = freshness_levels[freshness.get_active()].time
			policy.save_config()
			policy.recalculate()
		freshness.connect('changed', set_freshness)

		stable_toggle = widgets.get_widget('help_test')
		stable_toggle.set_active(policy.help_with_testing)
		def toggle_stability(toggle):
			policy.help_with_testing = toggle.get_active()
			policy.save_config()
			policy.recalculate()
		stable_toggle.connect('toggled', toggle_stability)

		# Keys
		keys_view = widgets.get_widget('trusted_keys')
		KeyList(keys_view)

		# Responses
		self.window.set_default_response(gtk.RESPONSE_CLOSE)
		self.window.default_widget.grab_focus()

		def response(dialog, resp):
			if resp in (gtk.RESPONSE_CLOSE, gtk.RESPONSE_DELETE_EVENT):
				self.window.destroy()
			elif resp == gtk.RESPONSE_HELP:
				gui_help.display()
		self.window.connect('response', response)

		self.window.set_default_size(-1, gtk.gdk.screen_height() / 3)

	def destroyed(self):
		global preferences_box
		preferences_box = None

class KeyList:
	def __init__(self, tv):
		self.trusted_keys = gtk.TreeStore(str, object)
		tv.set_model(self.trusted_keys)
		tc = gtk.TreeViewColumn(_('Trusted keys'), gtk.CellRendererText(), text = 0)
		tv.append_column(tc)
		trust.trust_db.ensure_uptodate()

		def update_keys():
			# Remember which ones are open
			expanded_elements = set()
			def add_row(tv, path):
				if len(path) == 1:
					domain = self.trusted_keys[path][0]
					expanded_elements.add(domain)
			tv.map_expanded_rows(add_row)

			self.trusted_keys.clear()
			domains = {}

			keys = gpg.load_keys(trust.trust_db.keys.keys())

			for fingerprint in keys:
				for domain in trust.trust_db.keys[fingerprint]:
					if domain not in domains:
						domains[domain] = set()
					domains[domain].add(keys[fingerprint])
			for domain in domains:
				iter = self.trusted_keys.append(None, [domain, None])
				for key in domains[domain]:
					self.trusted_keys.append(iter, [key.name, key])

			def may_expand(model, path, iter):
				if len(path) == 1:
					if model[iter][0] in expanded_elements:
						tv.expand_row(path, False)
			self.trusted_keys.foreach(may_expand)

		trust.trust_db.watchers.append(update_keys)
		tv.connect('destroy', lambda w: trust.trust_db.watchers.remove(update_keys))

		update_keys()

		def remove_key(fingerprint, domain):
			trust.trust_db.untrust_key(fingerprint, domain)
			trust.trust_db.notify()

		def trusted_keys_button_press(tv, bev):
			if bev.type == gtk.gdk.BUTTON_PRESS and bev.button == 3:
				pos = tv.get_path_at_pos(int(bev.x), int(bev.y))
				if not pos:
					return False
				path, col, x, y = pos
				if len(path) != 2:
					return False

				domain = self.trusted_keys[path[:-1]][0]
				key = self.trusted_keys[path][1]

				menu = gtk.Menu()

				item = gtk.MenuItem(_('Remove key for "%s"') % key.get_short_name())
				item.connect('activate',
					lambda item, fp = key.fingerprint, d = domain: remove_key(fp, d))
				item.show()
				menu.append(item)

				menu.popup(None, None, None, bev.button, bev.time)
				return True
			return False
		tv.connect('button-press-event', trusted_keys_button_press)

preferences_box = None
def show_preferences(policy):
	global preferences_box
	if preferences_box:
		preferences_box.destroy()
	preferences_box = Preferences(policy)
	preferences_box.window.show()

gui_help = help_box.HelpBox("Zero Install Preferences Help",
('Overview', """

There are three ways to control which implementations are chosen. You can adjust the \
network policy and the overall stability policy, which affect all interfaces, or you \
can edit the policy of individual interfaces."""),

('Network use', """
The 'Network use' option controls how the injector uses the network. If off-line, \
the network is not used at all. If 'Minimal' is selected then the injector will use \
the network if needed, but only if it has no choice. It will run an out-of-date \
version rather than download a newer one. If 'Full' is selected, the injector won't \
worry about how much it downloads, but will always pick the version it thinks is best."""),

('Freshness', """
The feed files, which provide the information about which versions are \
available, are also cached. To update them, click on 'Refresh all now'. You can also \
get the injector to check for new versions automatically from time to time using \
the Freshness setting."""),

('Help test new versions', """
The overall stability policy can either be to prefer stable versions, or to help test \
new versions. Choose whichever suits you. Since different programmers have different \
ideas of what 'stable' means, you may wish to override this on a per-interface basis.

To set the policy for an interface individually, select it in the main window and \
click on 'Interface Properties'. See that dialog's help text for more information."""),

('Security', """
This section lists all keys which you currently trust. When fetching a new program or \
updates for an existing one, the feed must be signed by one of these keys. If not, \
you will be prompted to confirm that you trust the new key, and it will then be added \
to this list. To remove a key, right-click on it and choose 'Remove' from the menu."""),
)
<|endoftext|>


<commit_before>import sys
import code

def comment(swc_path):
	f = open(swc_path, 'r')
	swc_lines = f.readlines()

	# add comment line to indicate soma
	soma_comment_index = -1
	for ii, line in enumerate(swc_lines):
		if line == "access sections[0]\n":
			soma_comment_index = ii - 1
			break
	swc_lines.insert(soma_comment_index, '// soma\n')

	# add comment line to indicate first branch
	branch1_comment_index = -1
	for ii, line in enumerate(swc_lines):
		if line == "access sections[1]\n":
			branch1_comment_index = ii - 1
			break
	swc_lines.insert(branch1_comment_index, '\n// d1')

	# save the updated hoc with the same path but altered name
	f = open(swc_path[:-4] + "_commented.hoc", 'w')
	for line in swc_lines:
		f.write(line)
	f.close()

if __name__ == "__main__":
	if len(sys.argv) != 2:
		print('\nhoc_commenter -- Written by Nathan Spencer 2016')
		print('Usage: python hoc_commenter.py "path/to/swc/file.swc"')
	else:
		comment(sys.argv[1])
<commit_msg>hoc commenter works dandily<commit_after>import code
import sys
import re

def comment(swc_path):
	f = open(swc_path, 'r')
	swc_lines = f.readlines()
	section_kidindex = { 1 : 1 }
	section_prefix = { 1 : 'd1,' }
	f.close()

	# add comment line to indicate soma
	soma_comment_index = -1
	for ii, line in enumerate(swc_lines):
		if line == "access sections[0]\n":
			soma_comment_index = ii - 1
			break
	swc_lines.insert(soma_comment_index, '// soma\n')

	# add comment line to indicate first branch
	branch1_comment_index = -1
	for ii, line in enumerate(swc_lines):
		if line == "access sections[1]\n":
			branch1_comment_index = ii - 1
			break
	swc_lines.insert(branch1_comment_index, '\n// d1')

	# set some key variables, create working copy of hoc text
	comment_offset = 2
	pattern = "connect sections\[([0-9]+)\]\(0\), sections\[([0-9]+)\]\(1\)"
	swc_lines_output = []
	for line in swc_lines:
		swc_lines_output.append(line)

	# add comments to the remaining branches
	for ii, line in enumerate(swc_lines):
		if(ii > branch1_comment_index + 4):
			results = re.search(pattern, line)
			if results:

				# write comment line for branch
				current = int(results.group(1))
				parent = int(results.group(2))
				comment = "// " + section_prefix[parent] + str(section_kidindex[parent]) + "\n"
				swc_lines_output.insert(ii - comment_offset, comment)

				# update dictionaries and offset
				section_prefix[current] = comment[3:-1] + ','
				section_kidindex[parent] = section_kidindex[parent] + 1
				section_kidindex[current] = 1
				comment_offset = comment_offset - 1

	# save the updated hoc with the same path but altered name
	f = open(swc_path[:-4] + "_commented.hoc", 'w')
	for line in swc_lines_output:
		f.write(line)
	f.close()

if __name__ == "__main__":
	if len(sys.argv) != 2:
		print('\nhoc_commenter -- Written by Nathan Spencer 2016')
		print('Usage: python hoc_commenter.py "path/to/swc/file.swc"')
	else:
		comment(sys.argv[1])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from twisted.internet import defer

from synapse.util.caches.descriptors import cached

from canonicaljson import encode_canonical_json
import ujson as json

from ._base import SQLBaseStore

class EndToEndKeyStore(SQLBaseStore):
    def set_e2e_device_keys(self, user_id, device_id, time_now, device_keys):
        """Stores device keys for a device. Returns whether there was a change
        or the keys were already in the database.
        """
        def _set_e2e_device_keys_txn(txn):
            old_key_json = self._simple_select_one_onecol_txn(
                txn,
                table="e2e_device_keys_json",
                keyvalues={
                    "user_id": user_id,
                    "device_id": device_id,
                },
                retcol="key_json",
                allow_none=True,
            )

            new_key_json = encode_canonical_json(device_keys)
            if old_key_json == new_key_json:
                return False

            self._simple_upsert_txn(
                txn,
                table="e2e_device_keys_json",
                keyvalues={
                    "user_id": user_id,
                    "device_id": device_id,
                },
                values={
                    "ts_added_ms": time_now,
                    "key_json": new_key_json,
                }
            )

            return True

        return self.runInteraction(
            "set_e2e_device_keys", _set_e2e_device_keys_txn
        )

    @defer.inlineCallbacks
    def get_e2e_device_keys(self, query_list, include_all_devices=False):
        """Fetch a list of device keys.
        Args:
            query_list(list): List of pairs of user_ids and device_ids.
            include_all_devices (bool): whether to include entries for devices
                that don't have device keys
        Returns:
            Dict mapping from user-id to dict mapping from device_id to
            dict containing "key_json", "device_display_name".
        """
        if not query_list:
            defer.returnValue({})

        results = yield self.runInteraction(
            "get_e2e_device_keys", self._get_e2e_device_keys_txn,
            query_list, include_all_devices,
        )

        for user_id, device_keys in results.iteritems():
            for device_id, device_info in device_keys.iteritems():
                device_info["keys"] = json.loads(device_info.pop("key_json"))

        defer.returnValue(results)

    def _get_e2e_device_keys_txn(self, txn, query_list, include_all_devices):
        query_clauses = []
        query_params = []

        for (user_id, device_id) in query_list:
            query_clause = "user_id = ?"
            query_params.append(user_id)

            if device_id is not None:
                query_clause += " AND device_id = ?"
                query_params.append(device_id)

            query_clauses.append(query_clause)

        sql = (
            "SELECT user_id, device_id, "
            "    d.display_name AS device_display_name, "
            "    k.key_json"
            " FROM devices d"
            "    %s JOIN e2e_device_keys_json k USING (user_id, device_id)"
            " WHERE %s"
        ) % (
            "LEFT" if include_all_devices else "INNER",
            " OR ".join("(" + q + ")" for q in query_clauses)
        )

        txn.execute(sql, query_params)
        rows = self.cursor_to_dict(txn)

        result = {}
        for row in rows:
            result.setdefault(row["user_id"], {})[row["device_id"]] = row

        return result

    @defer.inlineCallbacks
    def get_e2e_one_time_keys(self, user_id, device_id, key_ids):
        """Retrieve a number of one-time keys for a user

        Args:
            user_id(str): id of user to get keys for
            device_id(str): id of device to get keys for
            key_ids(list[str]): list of key ids (excluding algorithm) to
                retrieve

        Returns:
            deferred resolving to Dict[(str, str), str]: map from (algorithm,
            key_id) to json string for key
        """

        rows = yield self._simple_select_many_batch(
            table="e2e_one_time_keys_json",
            column="key_id",
            iterable=key_ids,
            retcols=("algorithm", "key_id", "key_json",),
            keyvalues={
                "user_id": user_id,
                "device_id": device_id,
            },
            desc="add_e2e_one_time_keys_check",
        )

        defer.returnValue({
            (row["algorithm"], row["key_id"]): row["key_json"] for row in rows
        })

    @defer.inlineCallbacks
    def add_e2e_one_time_keys(self, user_id, device_id, time_now, new_keys):
        """Insert some new one time keys for a device. Errors if any of the
        keys already exist.

        Args:
            user_id(str): id of user to get keys for
            device_id(str): id of device to get keys for
            time_now(long): insertion time to record (ms since epoch)
            new_keys(iterable[(str, str, str)]: keys to add - each a tuple of
                (algorithm, key_id, key json)
        """

        def _add_e2e_one_time_keys(txn):
            # We are protected from race between lookup and insertion due to
            # a unique constraint. If there is a race of two calls to
            # `add_e2e_one_time_keys` then they'll conflict and we will only
            # insert one set.
            self._simple_insert_many_txn(
                txn, table="e2e_one_time_keys_json",
                values=[
                    {
                        "user_id": user_id,
                        "device_id": device_id,
                        "algorithm": algorithm,
                        "key_id": key_id,
                        "ts_added_ms": time_now,
                        "key_json": json_bytes,
                    }
                    for algorithm, key_id, json_bytes in new_keys
                ],
            )
            self._invalidate_cache_and_stream(
                txn, self.count_e2e_one_time_keys, (user_id, device_id,)
            )
        yield self.runInteraction(
            "add_e2e_one_time_keys_insert", _add_e2e_one_time_keys
        )

    @cached(max_entries=10000)
    def count_e2e_one_time_keys(self, user_id, device_id):
        """ Count the number of one time keys the server has for a device
        Returns:
            Dict mapping from algorithm to number of keys for that algorithm.
        """
        def _count_e2e_one_time_keys(txn):
            sql = (
                "SELECT algorithm, COUNT(key_id) FROM e2e_one_time_keys_json"
                " WHERE user_id = ? AND device_id = ?"
                " GROUP BY algorithm"
            )
            txn.execute(sql, (user_id, device_id))
            result = {}
            for algorithm, key_count in txn:
                result[algorithm] = key_count
            return result
        return self.runInteraction(
            "count_e2e_one_time_keys", _count_e2e_one_time_keys
        )

    def claim_e2e_one_time_keys(self, query_list):
        """Take a list of one time keys out of the database"""
        def _claim_e2e_one_time_keys(txn):
            sql = (
                "SELECT key_id, key_json FROM e2e_one_time_keys_json"
                " WHERE user_id = ? AND device_id = ? AND algorithm = ?"
                " LIMIT 1"
            )
            result = {}
            delete = []
            for user_id, device_id, algorithm in query_list:
                user_result = result.setdefault(user_id, {})
                device_result = user_result.setdefault(device_id, {})
                txn.execute(sql, (user_id, device_id, algorithm))
                for key_id, key_json in txn:
                    device_result[algorithm + ":" + key_id] = key_json
                    delete.append((user_id, device_id, algorithm, key_id))
            sql = (
                "DELETE FROM e2e_one_time_keys_json"
                " WHERE user_id = ? AND device_id = ? AND algorithm = ?"
                " AND key_id = ?"
            )
            for user_id, device_id, algorithm, key_id in delete:
                txn.execute(sql, (user_id, device_id, algorithm, key_id))
                txn.call_after(
                    self.count_e2e_one_time_keys.invalidate, (user_id, device_id,)
                )
            return result
        return self.runInteraction(
            "claim_e2e_one_time_keys", _claim_e2e_one_time_keys
        )

    def delete_e2e_keys_by_device(self, user_id, device_id):
        def delete_e2e_keys_by_device_txn(txn):
            self._simple_delete_txn(
                txn,
                table="e2e_device_keys_json",
                keyvalues={"user_id": user_id, "device_id": device_id},
            )
            self._simple_delete_txn(
                txn,
                table="e2e_one_time_keys_json",
                keyvalues={"user_id": user_id, "device_id": device_id},
            )
            self._invalidate_cache_and_stream(
                txn, self.count_e2e_one_time_keys, (user_id, device_id,)
            )
        return self.runInteraction(
            "delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn
        )
<commit_msg>Missed an invalidation<commit_after># -*- coding: utf-8 -*-
# Copyright 2015, 2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from twisted.internet import defer

from synapse.util.caches.descriptors import cached

from canonicaljson import encode_canonical_json
import ujson as json

from ._base import SQLBaseStore

class EndToEndKeyStore(SQLBaseStore):
    def set_e2e_device_keys(self, user_id, device_id, time_now, device_keys):
        """Stores device keys for a device. Returns whether there was a change
        or the keys were already in the database.
        """
        def _set_e2e_device_keys_txn(txn):
            old_key_json = self._simple_select_one_onecol_txn(
                txn,
                table="e2e_device_keys_json",
                keyvalues={
                    "user_id": user_id,
                    "device_id": device_id,
                },
                retcol="key_json",
                allow_none=True,
            )

            new_key_json = encode_canonical_json(device_keys)
            if old_key_json == new_key_json:
                return False

            self._simple_upsert_txn(
                txn,
                table="e2e_device_keys_json",
                keyvalues={
                    "user_id": user_id,
                    "device_id": device_id,
                },
                values={
                    "ts_added_ms": time_now,
                    "key_json": new_key_json,
                }
            )

            return True

        return self.runInteraction(
            "set_e2e_device_keys", _set_e2e_device_keys_txn
        )

    @defer.inlineCallbacks
    def get_e2e_device_keys(self, query_list, include_all_devices=False):
        """Fetch a list of device keys.
        Args:
            query_list(list): List of pairs of user_ids and device_ids.
            include_all_devices (bool): whether to include entries for devices
                that don't have device keys
        Returns:
            Dict mapping from user-id to dict mapping from device_id to
            dict containing "key_json", "device_display_name".
        """
        if not query_list:
            defer.returnValue({})

        results = yield self.runInteraction(
            "get_e2e_device_keys", self._get_e2e_device_keys_txn,
            query_list, include_all_devices,
        )

        for user_id, device_keys in results.iteritems():
            for device_id, device_info in device_keys.iteritems():
                device_info["keys"] = json.loads(device_info.pop("key_json"))

        defer.returnValue(results)

    def _get_e2e_device_keys_txn(self, txn, query_list, include_all_devices):
        query_clauses = []
        query_params = []

        for (user_id, device_id) in query_list:
            query_clause = "user_id = ?"
            query_params.append(user_id)

            if device_id is not None:
                query_clause += " AND device_id = ?"
                query_params.append(device_id)

            query_clauses.append(query_clause)

        sql = (
            "SELECT user_id, device_id, "
            "    d.display_name AS device_display_name, "
            "    k.key_json"
            " FROM devices d"
            "    %s JOIN e2e_device_keys_json k USING (user_id, device_id)"
            " WHERE %s"
        ) % (
            "LEFT" if include_all_devices else "INNER",
            " OR ".join("(" + q + ")" for q in query_clauses)
        )

        txn.execute(sql, query_params)
        rows = self.cursor_to_dict(txn)

        result = {}
        for row in rows:
            result.setdefault(row["user_id"], {})[row["device_id"]] = row

        return result

    @defer.inlineCallbacks
    def get_e2e_one_time_keys(self, user_id, device_id, key_ids):
        """Retrieve a number of one-time keys for a user

        Args:
            user_id(str): id of user to get keys for
            device_id(str): id of device to get keys for
            key_ids(list[str]): list of key ids (excluding algorithm) to
                retrieve

        Returns:
            deferred resolving to Dict[(str, str), str]: map from (algorithm,
            key_id) to json string for key
        """

        rows = yield self._simple_select_many_batch(
            table="e2e_one_time_keys_json",
            column="key_id",
            iterable=key_ids,
            retcols=("algorithm", "key_id", "key_json",),
            keyvalues={
                "user_id": user_id,
                "device_id": device_id,
            },
            desc="add_e2e_one_time_keys_check",
        )

        defer.returnValue({
            (row["algorithm"], row["key_id"]): row["key_json"] for row in rows
        })

    @defer.inlineCallbacks
    def add_e2e_one_time_keys(self, user_id, device_id, time_now, new_keys):
        """Insert some new one time keys for a device. Errors if any of the
        keys already exist.

        Args:
            user_id(str): id of user to get keys for
            device_id(str): id of device to get keys for
            time_now(long): insertion time to record (ms since epoch)
            new_keys(iterable[(str, str, str)]: keys to add - each a tuple of
                (algorithm, key_id, key json)
        """

        def _add_e2e_one_time_keys(txn):
            # We are protected from race between lookup and insertion due to
            # a unique constraint. If there is a race of two calls to
            # `add_e2e_one_time_keys` then they'll conflict and we will only
            # insert one set.
            self._simple_insert_many_txn(
                txn, table="e2e_one_time_keys_json",
                values=[
                    {
                        "user_id": user_id,
                        "device_id": device_id,
                        "algorithm": algorithm,
                        "key_id": key_id,
                        "ts_added_ms": time_now,
                        "key_json": json_bytes,
                    }
                    for algorithm, key_id, json_bytes in new_keys
                ],
            )
            self._invalidate_cache_and_stream(
                txn, self.count_e2e_one_time_keys, (user_id, device_id,)
            )
        yield self.runInteraction(
            "add_e2e_one_time_keys_insert", _add_e2e_one_time_keys
        )

    @cached(max_entries=10000)
    def count_e2e_one_time_keys(self, user_id, device_id):
        """ Count the number of one time keys the server has for a device
        Returns:
            Dict mapping from algorithm to number of keys for that algorithm.
        """
        def _count_e2e_one_time_keys(txn):
            sql = (
                "SELECT algorithm, COUNT(key_id) FROM e2e_one_time_keys_json"
                " WHERE user_id = ? AND device_id = ?"
                " GROUP BY algorithm"
            )
            txn.execute(sql, (user_id, device_id))
            result = {}
            for algorithm, key_count in txn:
                result[algorithm] = key_count
            return result
        return self.runInteraction(
            "count_e2e_one_time_keys", _count_e2e_one_time_keys
        )

    def claim_e2e_one_time_keys(self, query_list):
        """Take a list of one time keys out of the database"""
        def _claim_e2e_one_time_keys(txn):
            sql = (
                "SELECT key_id, key_json FROM e2e_one_time_keys_json"
                " WHERE user_id = ? AND device_id = ? AND algorithm = ?"
                " LIMIT 1"
            )
            result = {}
            delete = []
            for user_id, device_id, algorithm in query_list:
                user_result = result.setdefault(user_id, {})
                device_result = user_result.setdefault(device_id, {})
                txn.execute(sql, (user_id, device_id, algorithm))
                for key_id, key_json in txn:
                    device_result[algorithm + ":" + key_id] = key_json
                    delete.append((user_id, device_id, algorithm, key_id))
            sql = (
                "DELETE FROM e2e_one_time_keys_json"
                " WHERE user_id = ? AND device_id = ? AND algorithm = ?"
                " AND key_id = ?"
            )
            for user_id, device_id, algorithm, key_id in delete:
                txn.execute(sql, (user_id, device_id, algorithm, key_id))
                txn.call_after(
                    self.count_e2e_one_time_keys.invalidate, (user_id, device_id,)
                )
                self._invalidate_cache_and_stream(
                    txn, self.count_e2e_one_time_keys, (user_id, device_id,)
                )
            return result
        return self.runInteraction(
            "claim_e2e_one_time_keys", _claim_e2e_one_time_keys
        )

    def delete_e2e_keys_by_device(self, user_id, device_id):
        def delete_e2e_keys_by_device_txn(txn):
            self._simple_delete_txn(
                txn,
                table="e2e_device_keys_json",
                keyvalues={"user_id": user_id, "device_id": device_id},
            )
            self._simple_delete_txn(
                txn,
                table="e2e_one_time_keys_json",
                keyvalues={"user_id": user_id, "device_id": device_id},
            )
            self._invalidate_cache_and_stream(
                txn, self.count_e2e_one_time_keys, (user_id, device_id,)
            )
        return self.runInteraction(
            "delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn
        )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import division, print_function, unicode_literals
import sys

class Namespace(object):
    """A simple class to hold the keys and aruments found from the
    input file.  This is a copy-and-paste job from the argparse module
    with some minor additions.

    You can populate the :py:class:`Namespace` at initiallization with
    a series of key-value pairs.  These are considered the defaults.
    """

    def __init__(self, **defaults):
        self._order = []
        self._defaults = defaults.copy()

    def __repr__(self):
        type_name = type(self).__name__
        arg_strings = []
        for name, value in self._get_kwargs():
            arg_strings.append('%s=%r' % (name, value))
        return '%s(%s)' % (type_name, ', '.join(arg_strings))

    def _get_kwargs(self):
        try:
            return [(k, getattr(self, k)) for k in self._order]
        except AttributeError:
            return sorted(self.__dict__.items())

    def __eq__(self, other):
        try:
            return vars(self) == vars(other)
        except TypeError:
            return vars(self) == other

    def __ne__(self, other):
        return not (self == other)

    def __iter__(self):
        for key in self.keys():
            yield key

    def __contains__(self, key):
        return key in self._order

    def __len__(self):
        return len(self._order)

    def add(self, key, val):
        """\
        Add a key-value pair to the :py:class:`Namespace`.

        :argument key:
            The key to add
        :type key: str
        :argument val:
            The value of the key
        :type val: anything
        """
        setattr(self, key, val)
        # Add this to the list of things found in the order found.
        # First check that it doesn't exist because duplicates can be
        # allowed
        if key not in self._order:
            self._order.append(key)
        # Remove this from the default dict
        try:
            del self._defaults[key]
        except KeyError:
            pass

    def remove(self, key):
        """\
        Remove a key from the :py:class:`Namespace`.

        :argument key:
            The key to remove.  If it does not exist, it is ignored.
        :type key: str
        """
        try:
            delattr(self, key)
        except AttributeError:
            pass
        try:
            self._order.remove(key)
        except ValueError:
            pass

    def get(self, key, default=None):
        """\
        Get the value of a key.  If the key does not exist,
        `default` is returned. This is alternative to the
        namespace.key syntax that does not raise an error
        when `key` does not exist.

        :argument key: The key whose value you wish to retrieve.
        :type key: str
        :argument default:
        :type default: anything (default is :py:obj:`None`)
        :returns: The value associated with `key`
        """
        return getattr(self, key, default)

    def keys(self):
        """\
        Just like the :py:meth:`dict.keys` function for the python
        :py:class:`dict`.

        :returns: :py:class:`tuple` of all keys in the :py:class:`Namespace`.
        """
        return tuple(self._order)

    def values(self):
        """\
        Just like the :py:meth:`dict.values` function for the python
        :py:class:`dict`.

        :returns: :py:class:`tuple` of all values in the
                  :py:class:`Namespace`.
        """
        return tuple([getattr(self, x) for x in self._order])

    def items(self):
        """\
        Just like the :py:meth:`dict.items` function for the python
        :py:class:`dict`.

        :returns: :py:class:`tuple` of :py:class:`tuple` s of
                  all key, value pairs in the :py:class:`Namespace`.
        """
        return tuple([(x, getattr(self, x)) for x in self._order])

    def finalize(self):
        """\
        Any defaults not yet added with the :py:meth:`add` are added
        to the :py:class:`Namespace`.
        """
        while True:
            try:
                key, val = self._defaults.popitem()
            except KeyError:
                break
            else:
                self.add(key, val)

class ReaderError(Exception):
    """\
    An exception for the :py:class:`InputReader` class.
    """
    def __init__(self, msg):
        self.msg = msg
    def __str__(self):
        return self.msg

class SUPPRESS(object):
    """
    Use this class to indicate that a key should be suppressed
    if not present (i.e. it does not appear in the input file).

    To use this, put :py:class:`SUPPRESS` for the default of a key.
    """
    pass
<commit_msg>Fixed typos in Namespace doc, and updated string formatting.<commit_after># -*- coding: utf-8 -*-
from __future__ import division, print_function, unicode_literals
import sys

class Namespace(object):
    """A simple class to hold the keys and arguments found from the
    input file.  This is a copy-and-paste job from the argparse module
    with some minor additions.

    You can populate the :py:class:`Namespace` at initialization with
    a series of key-value pairs.  These are considered the defaults.
    """

    def __init__(self, **defaults):
        self._order = []
        self._defaults = defaults.copy()

    def __repr__(self):
        type_name = type(self).__name__
        arg_strings = []
        for name, value in self._get_kwargs():
            arg_strings.append('{0}={1!r}'.format(name, value))
        return '{0}({1})'.format(type_name, ', '.join(arg_strings))

    def _get_kwargs(self):
        try:
            return [(k, getattr(self, k)) for k in self._order]
        except AttributeError:
            return sorted(self.__dict__.items())

    def __eq__(self, other):
        try:
            return vars(self) == vars(other)
        except TypeError:
            return vars(self) == other

    def __ne__(self, other):
        return not (self == other)

    def __iter__(self):
        for key in self.keys():
            yield key

    def __contains__(self, key):
        return key in self._order

    def __len__(self):
        return len(self._order)

    def add(self, key, val):
        """\
        Add a key-value pair to the :py:class:`Namespace`.

        :argument key:
            The key to add
        :type key: str
        :argument val:
            The value of the key
        :type val: anything
        """
        setattr(self, key, val)
        # Add this to the list of things found in the order found.
        # First check that it doesn't exist because duplicates can be
        # allowed
        if key not in self._order:
            self._order.append(key)
        # Remove this from the default dict
        try:
            del self._defaults[key]
        except KeyError:
            pass

    def remove(self, key):
        """\
        Remove a key from the :py:class:`Namespace`.

        :argument key:
            The key to remove.  If it does not exist, it is ignored.
        :type key: str
        """
        try:
            delattr(self, key)
        except AttributeError:
            pass
        try:
            self._order.remove(key)
        except ValueError:
            pass

    def get(self, key, default=None):
        """\
        Get the value of a key.  If the key does not exist,
        `default` is returned. This is alternative to the
        namespace.key syntax that does not raise an error
        when `key` does not exist.

        :argument key: The key whose value you wish to retrieve.
        :type key: str
        :argument default:
        :type default: anything (default is :py:obj:`None`)
        :returns: The value associated with `key`
        """
        return getattr(self, key, default)

    def keys(self):
        """\
        Just like the :py:meth:`dict.keys` function for the python
        :py:class:`dict`.

        :returns: :py:class:`tuple` of all keys in the :py:class:`Namespace`.
        """
        return tuple(self._order)

    def values(self):
        """\
        Just like the :py:meth:`dict.values` function for the python
        :py:class:`dict`.

        :returns: :py:class:`tuple` of all values in the
                  :py:class:`Namespace`.
        """
        return tuple([getattr(self, x) for x in self._order])

    def items(self):
        """\
        Just like the :py:meth:`dict.items` function for the python
        :py:class:`dict`.

        :returns: :py:class:`tuple` of :py:class:`tuple` s of
                  all key, value pairs in the :py:class:`Namespace`.
        """
        return tuple([(x, getattr(self, x)) for x in self._order])

    def finalize(self):
        """\
        Any defaults not yet added with the :py:meth:`add` are added
        to the :py:class:`Namespace`.
        """
        while True:
            try:
                key, val = self._defaults.popitem()
            except KeyError:
                break
            else:
                self.add(key, val)

class ReaderError(Exception):
    """\
    An exception for the :py:class:`InputReader` class.
    """
    def __init__(self, msg):
        self.msg = msg
    def __str__(self):
        return self.msg

class SUPPRESS(object):
    """
    Use this class to indicate that a key should be suppressed
    if not present (i.e. it does not appear in the input file).

    To use this, put :py:class:`SUPPRESS` for the default of a key.
    """
    pass
<|endoftext|>


<commit_before>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# https://www2.ed.gov/about/offices/list/oig/areports.html
archive = 1995

# options:
#   standard since/year options for a year range to fetch from.
#   report_id: limit to a single report
#
# Notes for IG's web team:
# - Fix the row for A17C0008 on
# http://www2.ed.gov/about/offices/list/oig/areports2003.html
# - Fix the published date for A06K0003
# on http://www2.ed.gov/about/offices/list/oig/areports2011.html
# - Multiple reports on http://www2.ed.gov/about/offices/list/oig/ireports.html
# say that they were published in 'Decemver' or 'Deccember' instead of 'December'

AUDIT_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/areports{}.html"
SEMIANNUAL_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/sarpages.html"

INSPECTION_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/aireports.html"
INVESTIGATIVE_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/ireports.html"
CONGRESSIONAL_TESTIMONY_URL = "https://www2.ed.gov/about/offices/list/oig/testimon.html"
SPECIAL_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/specialreportstocongress.html"
OTHER_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/otheroigproducts.html"

OTHER_REPORTS_URLS = [
  ("other", OTHER_REPORTS_URL),
  ("other", SPECIAL_REPORTS_URL),
  ("testimony", CONGRESSIONAL_TESTIMONY_URL),
  ("investigation", INVESTIGATIVE_REPORTS_URL),
  ("inspection", INSPECTION_REPORTS_URL),
]

REPORT_PUBLISHED_MAP = {
  "statelocal032002": datetime.datetime(2002, 3, 21),
  "statloc082001": datetime.datetime(2001, 8, 3),
  "A17B0006": datetime.datetime(2002, 2, 27),
  "A17A0002": datetime.datetime(2001, 2, 28),
  "A1790019": datetime.datetime(2000, 2, 28),  # Approximation
  "A17C0008": datetime.datetime(2003, 1, 31),
  "PESMemo": datetime.datetime(2001, 1, 1),  # Approximation
  "s1370001": datetime.datetime(1999, 3, 18),
  "oigqualitystandardsforalternativeproducts": datetime.datetime(2010, 3, 11),
}

def run(options):
  year_range = inspector.year_range(options, archive)

  # optional: limit to a single report
  report_id = options.get("report_id")

  # Get the audit reports
  pre_1998_audit_flag = False
  for year in year_range:
    if year <= 1998:
      if pre_1998_audit_flag:
        continue
      else:
        pre_1998_audit_flag = True
    url = audit_url_for(year)
    doc = utils.beautifulsoup_from_url(url)
    agency_tables = doc.find_all("table", {"border": 1})
    if not agency_tables:
      raise inspector.NoReportsFoundError("Department of Education (%d audit reports)" % year)
    for agency_table in agency_tables:
      results = agency_table.select("tr")
      for index, result in enumerate(results):
        if not index:
          # First row is the header
          continue
        report = audit_report_from(result, url, year_range)
        if report:
          # optional: filter to a single report
          if report_id and (report_id != report['report_id']):
            continue

          inspector.save_report(report)

  # Get semiannual reports
  doc = utils.beautifulsoup_from_url(SEMIANNUAL_REPORTS_URL)
  table = doc.find("table", {"border": 1})
  for index, result in enumerate(table.select("tr")):
    if index < 2:
      # The first two rows are headers
      continue
    report = semiannual_report_from(result, SEMIANNUAL_REPORTS_URL, year_range)
    if report:
      # optional: filter to a single report
      if report_id and (report_id != report['report_id']):
        continue
      inspector.save_report(report)

  # Get other reports
  for report_type, url in OTHER_REPORTS_URLS:
    doc = utils.beautifulsoup_from_url(url)
    results = doc.select("div.contentText ul li")
    if not results:
      raise inspector.NoReportsFoundError("Department of Education (%s)" % report_type)
    for result in results:
      report = report_from(result, url, report_type, year_range)
      if report:
        # optional: filter to a single report
        if report_id and (report_id != report['report_id']):
          continue

        inspector.save_report(report)

audit_reports_seen = set()

def audit_report_from(result, page_url, year_range):
  if not result.text.strip():
    # Just an empty row
    return

  title = result.select("td")[0].text.strip().replace('\n', '')
  if title == "Enterprise Architecture.":
    title = "Audit of Enterprise Architecture."
  if title == "The Department of Education's process for identifying and " \
          "Monitoring High-Risk Contracts that Support Office of Educational " \
          "Research and Improvement Programs.":
    title = "Audit of The Department of Education's process for identifying " \
        "and Monitoring High-Risk Contracts that Support Office of " \
        "Educational Research and Improvement (OERI) Programs."

  report_url = urljoin(page_url, result.select("td a")[0].get('href'))
  if report_url.startswith("http://www.ed.gov/"):
    report_url = report_url.replace("http://www.ed.gov/", "https://www2.ed.gov/")

  report_id = None
  if len(result.select("td")) != 3:
    report_id = result.select("td")[1].text.strip()
    if report_id.startswith("ACN: "):
      report_id = report_id[5:]
  if not report_id:
    report_filename = report_url.split("/")[-1]
    report_id, extension = os.path.splitext(report_filename)

  if report_id in REPORT_PUBLISHED_MAP:
    published_on = REPORT_PUBLISHED_MAP[report_id]
  else:
    # See notes to the IG Web team for some of this
    published_on_text = result.select("td")[2].text.strip().replace(")", "").replace("//", "/")
    date_formats = ['%m/%d/%Y', '%m/%d/%y', '%m/%Y']
    published_on = None
    for date_format in date_formats:
      try:
        published_on = datetime.datetime.strptime(published_on_text, date_format)
      except ValueError:
        pass

  # The following report is linked to twice from the IG's website, once with
  # the date 02/06/2003 (correct) and once with the date 02/06/2002. Both links
  # point to the same PDF file, have the same title, and have the same ACN.
  # If we are processing the duplicate entry, suppress it.
  if report_id == "A17D0002" and published_on.year == 2002 and published_on.month == 2 and published_on.day == 6:
    return

  # These files are of the same report, but one is higher quality and has a TOC
  if report_url == "https://www2.ed.gov/about/offices/list/ocfo/FY2001AccountabilityRpt.pdf":
    report_url = "https://www2.ed.gov/about/offices/list/ocfo/FY2001AccountabilityReport.pdf"

  # The ACN "B19I0001" is used for two different reports
  if report_url == "https://www2.ed.gov/about/offices/list/oig/auditreports/fy2008/b19i0001a.pdf":
    report_id = "B19I0001A"
  elif report_url == "https://www2.ed.gov/about/offices/list/oig/auditreports/fy2008/b19i0001p.pdf":
    report_id = "B19I0001P"

  # The federal student aid site has been reorganized
  if report_url == "http://www.federalstudentaid.ed.gov/docs/fsa_annual_report_2008.pdf":
    report_url = "https://studentaid.ed.gov/sa/sites/default/files/fsawg/static/gw/docs/fsa_annual_report_2008.pdf"

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  key = (report_id, title, report_url)
  if key in audit_reports_seen:
    return
  audit_reports_seen.add(key)

  if "thestartingline.ed.gov" in report_url:
    return None

  report = {
    'inspector': 'education',
    'inspector_url': 'https://www2.ed.gov/about/offices/list/oig/',
    'agency': 'education',
    'agency_name': "Department of Education",
    'type': 'audit',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }

  # This report was taken down; the URL now returns a 404
  if report_url == "https://www2.ed.gov/about/offices/list/oig/auditreports/a07c0001.pdf":
    del report['url']
    report['unreleased'] = True
    report['missing'] = True
    report['landing_url'] = page_url

  return report

def audit_url_for(year):
  if year < 1998:
    # This is the first listed year
    year = 1998

  if year == 2001:
    # This one needs a capital A
    return "https://www2.ed.gov/about/offices/list/oig/Areports2001.html"

  if year == datetime.datetime.today().year:
    # The current year is on the main page
    return AUDIT_REPORTS_URL.format("")
  else:
    return AUDIT_REPORTS_URL.format(year)

def semiannual_report_from(result, page_url, year_range):
  report_url = urljoin(page_url, result.select("a")[0].get('href'))
  report_filename = report_url.split("/")[-1]
  report_id, extension = os.path.splitext(report_filename)
  date_range_text = result.select("td")[1].text
  title = "Semiannual Report - {}".format(date_range_text)
  published_on_text = date_range_text.split("-")[-1].strip()
  published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  if "thestartingline.ed.gov" in report_url:
    return None

  report = {
    'inspector': 'education',
    'inspector_url': 'https://www2.ed.gov/about/offices/list/oig/',
    'agency': 'education',
    'agency_name': "Department of Education",
    'type': 'semiannual_report',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

other_reports_seen = set()

def report_from(result, url, report_type, year_range):
  report_url = urljoin(url, result.select("a")[0].get('href'))
  report_filename = report_url.split("/")[-1]
  report_id, extension = os.path.splitext(report_filename)

  title = result.text.split(" ACN:")[0]
  if report_id in REPORT_PUBLISHED_MAP:
    published_on = REPORT_PUBLISHED_MAP[report_id]
  else:
    result_text = result.text.replace(",", "").replace("\n", " ")
    result_text = " ".join(result_text.split())  # Remove any double spaces
    result_text = result_text.replace("Decemver", "December").replace("Deccember", "December")  # See note to IG Web team
    result_text = result_text.replace("//", "/")
    try:
      published_on_text = "/".join(re.search("(\d+)/(\d+)/(\d+)", result_text).groups())
      published_on = datetime.datetime.strptime(published_on_text, '%m/%d/%Y')
    except AttributeError:
      try:
        published_on_text = "/".join(re.search("(\d+)/(\d+)", result_text).groups())
        published_on = datetime.datetime.strptime(published_on_text, '%m/%Y')
      except AttributeError:
        try:
          published_on_text = "/".join(re.search("(\w+) (\d+) (\d+)", result_text).groups())
          published_on = datetime.datetime.strptime(published_on_text, '%B/%d/%Y')
        except AttributeError:
          published_on_text = "/".join(re.search("(\w+) (\d+)", result_text).groups())
          published_on = datetime.datetime.strptime(published_on_text, '%B/%Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  if "thestartingline.ed.gov" in report_url:
    return None

  if (report_id in ("x13j0003", "x19k0008", "x05j0019") and
          url == OTHER_REPORTS_URL):
    # These reports are also posted on the audit report pages, so we skip the
    # copy on the other reports page
    return None

  key = (report_id, title, report_url)
  if key in other_reports_seen:
    return
  other_reports_seen.add(key)

  report = {
    'inspector': 'education',
    'inspector_url': 'https://www2.ed.gov/about/offices/list/oig/',
    'agency': 'education',
    'agency_name': "Department of Education",
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }

  # This report comes up as 404, mark missing
  if report_url == "https://www2.ed.gov/about/offices/list/oig/invtreports/ga032013.html":
    del report['url']
    report['unreleased'] = True
    report['missing'] = True
    report['landing_url'] = url

  return report

utils.run(run) if (__name__ == "__main__") else None
<commit_msg>[education] Fix semiannual reports<commit_after>#!/usr/bin/env python

import datetime
import logging
import os
import re
from urllib.parse import urljoin

from utils import utils, inspector

# https://www2.ed.gov/about/offices/list/oig/areports.html
archive = 1995

# options:
#   standard since/year options for a year range to fetch from.
#   report_id: limit to a single report
#
# Notes for IG's web team:
# - Fix the row for A17C0008 on
# http://www2.ed.gov/about/offices/list/oig/areports2003.html
# - Fix the published date for A06K0003
# on http://www2.ed.gov/about/offices/list/oig/areports2011.html
# - Multiple reports on http://www2.ed.gov/about/offices/list/oig/ireports.html
# say that they were published in 'Decemver' or 'Deccember' instead of 'December'

AUDIT_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/areports{}.html"
SEMIANNUAL_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/sarpages.html"

INSPECTION_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/aireports.html"
INVESTIGATIVE_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/ireports.html"
CONGRESSIONAL_TESTIMONY_URL = "https://www2.ed.gov/about/offices/list/oig/testimon.html"
SPECIAL_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/specialreportstocongress.html"
OTHER_REPORTS_URL = "https://www2.ed.gov/about/offices/list/oig/otheroigproducts.html"

OTHER_REPORTS_URLS = [
  ("other", OTHER_REPORTS_URL),
  ("other", SPECIAL_REPORTS_URL),
  ("testimony", CONGRESSIONAL_TESTIMONY_URL),
  ("investigation", INVESTIGATIVE_REPORTS_URL),
  ("inspection", INSPECTION_REPORTS_URL),
]

REPORT_PUBLISHED_MAP = {
  "statelocal032002": datetime.datetime(2002, 3, 21),
  "statloc082001": datetime.datetime(2001, 8, 3),
  "A17B0006": datetime.datetime(2002, 2, 27),
  "A17A0002": datetime.datetime(2001, 2, 28),
  "A1790019": datetime.datetime(2000, 2, 28),  # Approximation
  "A17C0008": datetime.datetime(2003, 1, 31),
  "PESMemo": datetime.datetime(2001, 1, 1),  # Approximation
  "s1370001": datetime.datetime(1999, 3, 18),
  "oigqualitystandardsforalternativeproducts": datetime.datetime(2010, 3, 11),
}

def run(options):
  year_range = inspector.year_range(options, archive)

  # optional: limit to a single report
  report_id = options.get("report_id")

  # Get the audit reports
  pre_1998_audit_flag = False
  for year in year_range:
    if year <= 1998:
      if pre_1998_audit_flag:
        continue
      else:
        pre_1998_audit_flag = True
    url = audit_url_for(year)
    doc = utils.beautifulsoup_from_url(url)
    agency_tables = doc.find_all("table", {"border": 1})
    if not agency_tables:
      raise inspector.NoReportsFoundError("Department of Education (%d audit reports)" % year)
    for agency_table in agency_tables:
      results = agency_table.select("tr")
      for index, result in enumerate(results):
        if not index:
          # First row is the header
          continue
        report = audit_report_from(result, url, year_range)
        if report:
          # optional: filter to a single report
          if report_id and (report_id != report['report_id']):
            continue

          inspector.save_report(report)

  # Get semiannual reports
  doc = utils.beautifulsoup_from_url(SEMIANNUAL_REPORTS_URL)
  table = doc.find("table", {"border": 1})
  for index, result in enumerate(table.select("tr")):
    if index < 2:
      # The first two rows are headers
      continue
    report = semiannual_report_from(result, SEMIANNUAL_REPORTS_URL, year_range)
    if report:
      # optional: filter to a single report
      if report_id and (report_id != report['report_id']):
        continue
      inspector.save_report(report)

  # Get other reports
  for report_type, url in OTHER_REPORTS_URLS:
    doc = utils.beautifulsoup_from_url(url)
    results = doc.select("div.contentText ul li")
    if not results:
      raise inspector.NoReportsFoundError("Department of Education (%s)" % report_type)
    for result in results:
      report = report_from(result, url, report_type, year_range)
      if report:
        # optional: filter to a single report
        if report_id and (report_id != report['report_id']):
          continue

        inspector.save_report(report)

audit_reports_seen = set()

def audit_report_from(result, page_url, year_range):
  if not result.text.strip():
    # Just an empty row
    return

  title = result.select("td")[0].text.strip().replace('\n', '')
  if title == "Enterprise Architecture.":
    title = "Audit of Enterprise Architecture."
  if title == "The Department of Education's process for identifying and " \
          "Monitoring High-Risk Contracts that Support Office of Educational " \
          "Research and Improvement Programs.":
    title = "Audit of The Department of Education's process for identifying " \
        "and Monitoring High-Risk Contracts that Support Office of " \
        "Educational Research and Improvement (OERI) Programs."

  report_url = urljoin(page_url, result.select("td a")[0].get('href'))
  if report_url.startswith("http://www.ed.gov/"):
    report_url = report_url.replace("http://www.ed.gov/", "https://www2.ed.gov/")

  report_id = None
  if len(result.select("td")) != 3:
    report_id = result.select("td")[1].text.strip()
    if report_id.startswith("ACN: "):
      report_id = report_id[5:]
  if not report_id:
    report_filename = report_url.split("/")[-1]
    report_id, extension = os.path.splitext(report_filename)

  if report_id in REPORT_PUBLISHED_MAP:
    published_on = REPORT_PUBLISHED_MAP[report_id]
  else:
    # See notes to the IG Web team for some of this
    published_on_text = result.select("td")[2].text.strip().replace(")", "").replace("//", "/")
    date_formats = ['%m/%d/%Y', '%m/%d/%y', '%m/%Y']
    published_on = None
    for date_format in date_formats:
      try:
        published_on = datetime.datetime.strptime(published_on_text, date_format)
      except ValueError:
        pass

  # The following report is linked to twice from the IG's website, once with
  # the date 02/06/2003 (correct) and once with the date 02/06/2002. Both links
  # point to the same PDF file, have the same title, and have the same ACN.
  # If we are processing the duplicate entry, suppress it.
  if report_id == "A17D0002" and published_on.year == 2002 and published_on.month == 2 and published_on.day == 6:
    return

  # These files are of the same report, but one is higher quality and has a TOC
  if report_url == "https://www2.ed.gov/about/offices/list/ocfo/FY2001AccountabilityRpt.pdf":
    report_url = "https://www2.ed.gov/about/offices/list/ocfo/FY2001AccountabilityReport.pdf"

  # The ACN "B19I0001" is used for two different reports
  if report_url == "https://www2.ed.gov/about/offices/list/oig/auditreports/fy2008/b19i0001a.pdf":
    report_id = "B19I0001A"
  elif report_url == "https://www2.ed.gov/about/offices/list/oig/auditreports/fy2008/b19i0001p.pdf":
    report_id = "B19I0001P"

  # The federal student aid site has been reorganized
  if report_url == "http://www.federalstudentaid.ed.gov/docs/fsa_annual_report_2008.pdf":
    report_url = "https://studentaid.ed.gov/sa/sites/default/files/fsawg/static/gw/docs/fsa_annual_report_2008.pdf"

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  key = (report_id, title, report_url)
  if key in audit_reports_seen:
    return
  audit_reports_seen.add(key)

  if "thestartingline.ed.gov" in report_url:
    return None

  report = {
    'inspector': 'education',
    'inspector_url': 'https://www2.ed.gov/about/offices/list/oig/',
    'agency': 'education',
    'agency_name': "Department of Education",
    'type': 'audit',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }

  # This report was taken down; the URL now returns a 404
  if report_url == "https://www2.ed.gov/about/offices/list/oig/auditreports/a07c0001.pdf":
    del report['url']
    report['unreleased'] = True
    report['missing'] = True
    report['landing_url'] = page_url

  return report

def audit_url_for(year):
  if year < 1998:
    # This is the first listed year
    year = 1998

  if year == 2001:
    # This one needs a capital A
    return "https://www2.ed.gov/about/offices/list/oig/Areports2001.html"

  if year == datetime.datetime.today().year:
    # The current year is on the main page
    return AUDIT_REPORTS_URL.format("")
  else:
    return AUDIT_REPORTS_URL.format(year)

def semiannual_report_from(result, page_url, year_range):
  report_url = urljoin(page_url, result.select("a")[0].get('href'))
  report_filename = report_url.split("/")[-1]
  report_id, extension = os.path.splitext(report_filename)
  date_range_text = result.select("td")[1].text
  title = "Semiannual Report - {}".format(date_range_text)
  published_on_text = date_range_text.split("through")[-1].strip()
  published_on = datetime.datetime.strptime(published_on_text, '%B %d, %Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  if "thestartingline.ed.gov" in report_url:
    return None

  report = {
    'inspector': 'education',
    'inspector_url': 'https://www2.ed.gov/about/offices/list/oig/',
    'agency': 'education',
    'agency_name': "Department of Education",
    'type': 'semiannual_report',
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }
  return report

other_reports_seen = set()

def report_from(result, url, report_type, year_range):
  report_url = urljoin(url, result.select("a")[0].get('href'))
  report_filename = report_url.split("/")[-1]
  report_id, extension = os.path.splitext(report_filename)

  title = result.text.split(" ACN:")[0]
  if report_id in REPORT_PUBLISHED_MAP:
    published_on = REPORT_PUBLISHED_MAP[report_id]
  else:
    result_text = result.text.replace(",", "").replace("\n", " ")
    result_text = " ".join(result_text.split())  # Remove any double spaces
    result_text = result_text.replace("Decemver", "December").replace("Deccember", "December")  # See note to IG Web team
    result_text = result_text.replace("//", "/")
    try:
      published_on_text = "/".join(re.search("(\d+)/(\d+)/(\d+)", result_text).groups())
      published_on = datetime.datetime.strptime(published_on_text, '%m/%d/%Y')
    except AttributeError:
      try:
        published_on_text = "/".join(re.search("(\d+)/(\d+)", result_text).groups())
        published_on = datetime.datetime.strptime(published_on_text, '%m/%Y')
      except AttributeError:
        try:
          published_on_text = "/".join(re.search("(\w+) (\d+) (\d+)", result_text).groups())
          published_on = datetime.datetime.strptime(published_on_text, '%B/%d/%Y')
        except AttributeError:
          published_on_text = "/".join(re.search("(\w+) (\d+)", result_text).groups())
          published_on = datetime.datetime.strptime(published_on_text, '%B/%Y')

  if published_on.year not in year_range:
    logging.debug("[%s] Skipping, not in requested range." % report_url)
    return

  if "thestartingline.ed.gov" in report_url:
    return None

  if (report_id in ("x13j0003", "x19k0008", "x05j0019") and
          url == OTHER_REPORTS_URL):
    # These reports are also posted on the audit report pages, so we skip the
    # copy on the other reports page
    return None

  key = (report_id, title, report_url)
  if key in other_reports_seen:
    return
  other_reports_seen.add(key)

  report = {
    'inspector': 'education',
    'inspector_url': 'https://www2.ed.gov/about/offices/list/oig/',
    'agency': 'education',
    'agency_name': "Department of Education",
    'type': report_type,
    'report_id': report_id,
    'url': report_url,
    'title': title,
    'published_on': datetime.datetime.strftime(published_on, "%Y-%m-%d"),
  }

  # This report comes up as 404, mark missing
  if report_url == "https://www2.ed.gov/about/offices/list/oig/invtreports/ga032013.html":
    del report['url']
    report['unreleased'] = True
    report['missing'] = True
    report['landing_url'] = url

  return report

utils.run(run) if (__name__ == "__main__") else None
<|endoftext|>