<commit_before>import sublime, sublime_plugin
import os
import sys
import subprocess
import threading
import tempfile
from queue import Queue, Empty

def view_is_at_bottom(view):
    layout_h = view.layout_extent()[1]
    view_h = view.viewport_extent()[1]
    view_y = view.viewport_position()[1]
    line_h = view.line_height()

    view_taller_than_content = layout_h <= view_h
    at_bottom_of_content = view_y + view_h >= layout_h - line_h

    return view_taller_than_content or at_bottom_of_content

def plugin_unloaded():
    SuperColliderProcess.stop()
    SuperColliderProcess.deactivate_post_view('SublimeText: sclang terminated!\n')

class SuperColliderProcess():
    settings = sublime.load_settings("SuperCollider.sublime-settings")
    sclang_thread = None
    sclang_process = None
    sclang_queue = None
    sclang_thread = None

    post_view_name = 'SuperCollider - Post'
    post_view = None
    post_view_cache = None
    post_view_max_lines = None
    # the post view buffer is kept alive, even when the views into it are closed
    # with this cache, we can restore the previous state when re-opening the
    # window
    # Just trying to pull the content from the view doesn't work as sometimes
    # content is empty (problem with async timeout? being garbage collected?)
    # Instead using an explicit cache, updated lazily when view is closed and
    # new view being opened

    def update_post_view_max_lines():
        SuperColliderProcess.post_view_max_lines = SuperColliderProcess.settings.get('max_post_view_lines')

    def start():
        if SuperColliderProcess.is_alive():
            sublime.status_message("sclang already running!")
            return

        # load settings
        sc_dir = SuperColliderProcess.settings.get("sc_dir")
        sc_exe = SuperColliderProcess.settings.get("sc_exe")
        SuperColliderProcess.update_post_view_max_lines()
        SuperColliderProcess.settings.add_on_change('max_post_view_lines',
            SuperColliderProcess.update_post_view_max_lines)

        # create subprocess
        path = None
        cwd = None
        close_fds = None
        shell = None

        if os.name is 'posix':
            path = sc_dir + sc_exe
            close_fds = True
            shell = False
        else:
            path = sc_exe
            cwd = sc_dir
            close_fds = False
            shell = True

        SuperColliderProcess.sclang_process = subprocess.Popen(
            args = [path, '-i', 'sublime'],
            cwd = cwd,
            bufsize = 0,
            stdin = subprocess.PIPE,
            stdout = subprocess.PIPE,
            stderr = subprocess.STDOUT,
            close_fds = close_fds,
            shell = shell
        )

        # create post window temp file
        SuperColliderProcess.open_post_view()

        # create post window update queue and thread
        def enqueue_output(input, queue):
            for line in iter(input.readline, b''):
                queue.put(line.decode('utf-8'))
            input.close()
            if SuperColliderProcess.has_post_view():
                SuperColliderProcess.deactivate_post_view('SublimeText: sclang terminated!\n')

        SuperColliderProcess.sclang_queue = Queue()
        SuperColliderProcess.sclang_thread = threading.Thread(
            target = enqueue_output,
            args = (
                SuperColliderProcess.sclang_process.stdout,
                SuperColliderProcess.sclang_queue
            )
        )

        SuperColliderProcess.sclang_thread.daemon = True # dies with the program
        SuperColliderProcess.sclang_thread.start()
        sublime.status_message("Starting SuperCollider")

    def stop():
        if SuperColliderProcess.is_alive():
            SuperColliderProcess.execute("0.exit;")
        else:
            sublime.status_message("sclang not running")

    def is_alive():
        return (SuperColliderProcess.sclang_thread is not None and
                SuperColliderProcess.sclang_thread.isAlive())

    def execute(cmd):
        if SuperColliderProcess.sclang_process is None:
            sublime.status_message("sclang not running")
            return

        SuperColliderProcess.sclang_process.stdin.write(bytes(cmd + '\x0c', 'utf-8'))
        SuperColliderProcess.sclang_process.stdin.flush()

    def has_post_view():
        return SuperColliderProcess.post_view is not None

    def post_view_id():
        if SuperColliderProcess.has_post_view():
            return SuperColliderProcess.post_view.id()
        else:
            return None

    def update_post_view():
        if SuperColliderProcess.is_alive() and SuperColliderProcess.has_post_view():
            try:
                line = SuperColliderProcess.sclang_queue.get_nowait()
            except Empty:
                pass
            else:
                SuperColliderProcess.post_view.run_command('super_collider_update_post_view', {
                    'content': line,
                    'max_lines': SuperColliderProcess.post_view_max_lines
                })

        sublime.set_timeout(SuperColliderProcess.update_post_view, 20)

    def cache_post_view(content):
        SuperColliderProcess.post_view_cache = content

    def deactivate_post_view(msg):
        if SuperColliderProcess.has_post_view():
            SuperColliderProcess.post_view.run_command('super_collider_update_post_view', {
                'content': msg,
                'force_scroll': True
            })
            SuperColliderProcess.post_view.set_name(SuperColliderProcess.post_view_name + ' - Inactive')
            SuperColliderProcess.post_view = None

    def open_post_view():
        if len(sublime.windows()) is 0:
            sublime.run_command('new_window')

        old_view = None

        if SuperColliderProcess.has_post_view():
            old_view = SuperColliderProcess.post_view
            # switch to the post window if it is in the current window
            for window in sublime.windows():
                if window.views().count(old_view):
                    window.focus_view(old_view)
                    return
                else:
                    # cache old post view contents
                    content = old_view.substr(sublime.Region(0, old_view.size()))
                    SuperColliderProcess.cache_post_view(content)

        window = sublime.active_window()

        # create new post view
        post_view = window.new_file()
        post_view.set_name(SuperColliderProcess.post_view_name)
        post_view.set_scratch(True)
        post_view.settings().set('rulers', 0)
        post_view.settings().set('line_numbers', False)

        # update the view with previoius view content if possible
        if SuperColliderProcess.post_view_cache is not None:
            post_view.run_command('super_collider_update_post_view', {
                'content': SuperColliderProcess.post_view_cache,
                'max_lines': SuperColliderProcess.post_view_max_lines,
                'force_scroll': True
            })
            SuperColliderProcess.post_view_cache = None

        # deactivate old view
        if old_view is not None:
            SuperColliderProcess.deactivate_post_view('Sublime Text: This post window now deactivated!\n')

        # update post view to newly created
        SuperColliderProcess.post_view = post_view
        SuperColliderProcess.update_post_view()

class SuperColliderStartCommand(sublime_plugin.ApplicationCommand):
    def run(self):
        SuperColliderProcess.start()

class SuperColliderStopCommand(sublime_plugin.ApplicationCommand):
    def run(self):
        SuperColliderProcess.stop()

class SuperColliderUpdatePostViewCommand(sublime_plugin.TextCommand):
    def run(self, edit, content, max_lines=-1, force_scroll=False):
        # insert text
        self.view.insert(edit, self.view.size(), content)

        # erase overspill
        all_lines = self.view.lines(sublime.Region(0, self.view.size()))
        total_lines = len(all_lines)
        if total_lines > max_lines and max_lines >= 1:
            end = all_lines[total_lines - max_lines].b + 1
            self.view.erase(edit, sublime.Region(0, end))

        # scroll
        if force_scroll or view_is_at_bottom(self.view):
            self.view.show(self.view.size())

class SuperColliderOpenPostViewCommand(sublime_plugin.ApplicationCommand):
    def run(self):
        SuperColliderProcess.open_post_view()

class SuperColliderListener(sublime_plugin.EventListener):
    def on_close(self, view):
        if view.id() is SuperColliderProcess.post_view_id():
            content = view.substr(sublime.Region(0, view.size()))
            SuperColliderProcess.cache_post_view(content)

class SuperColliderSendCommand(sublime_plugin.TextCommand):
    def run(self, edit):
        for sel in self.view.sel():
            cmd = None
            # "selection" is a single point
            if sel.a == sel.b:
                cmd = self.view.substr(self.view.line(sel))
            else:
                cmd = self.view.substr(sel)

            SuperColliderProcess.execute(cmd)

class SuperColliderLoop(sublime_plugin.ApplicationCommand):
    def run(self):
        SuperColliderProcess.execute("{inf.do{|x| x.postln; 0.1.wait; }}.fork")

class SuperColliderTest(sublime_plugin.ApplicationCommand):
    def run(self, count):
        SuperColliderProcess.execute(str(count) + ".do {|i| i.postln; };")

# TODO if re-open last tab and context SC, then open new post window
# TODO option on where to open post window: new tab, new group, new window, terminal
# TODO refocus on original window on open_post_view
# TODO re-write with plugin_loaded and process instance?<commit_msg>hardcore check if post_view exists<commit_after>import sublime, sublime_plugin
import os
import sys
import subprocess
import threading
import tempfile
from queue import Queue, Empty

def view_is_at_bottom(view):
    layout_h = view.layout_extent()[1]
    view_h = view.viewport_extent()[1]
    view_y = view.viewport_position()[1]
    line_h = view.line_height()

    view_taller_than_content = layout_h <= view_h
    at_bottom_of_content = view_y + view_h >= layout_h - line_h

    return view_taller_than_content or at_bottom_of_content

def plugin_unloaded():
    SuperColliderProcess.stop()
    SuperColliderProcess.deactivate_post_view('SublimeText: sclang terminated!\n')

class SuperColliderProcess():
    settings = sublime.load_settings("SuperCollider.sublime-settings")
    sclang_thread = None
    sclang_process = None
    sclang_queue = None
    sclang_thread = None

    post_view_name = 'SuperCollider - Post'
    post_view = None
    post_view_cache = None
    post_view_max_lines = None
    # the post view buffer is kept alive, even when the views into it are closed
    # with this cache, we can restore the previous state when re-opening the
    # window
    # Just trying to pull the content from the view doesn't work as sometimes
    # content is empty (problem with async timeout? being garbage collected?)
    # Instead using an explicit cache, updated lazily when view is closed and
    # new view being opened

    def update_post_view_max_lines():
        SuperColliderProcess.post_view_max_lines = SuperColliderProcess.settings.get('max_post_view_lines')

    def start():
        if SuperColliderProcess.is_alive():
            sublime.status_message("sclang already running!")
            return

        # load settings
        sc_dir = SuperColliderProcess.settings.get("sc_dir")
        sc_exe = SuperColliderProcess.settings.get("sc_exe")
        SuperColliderProcess.update_post_view_max_lines()
        SuperColliderProcess.settings.add_on_change('max_post_view_lines',
            SuperColliderProcess.update_post_view_max_lines)

        # create subprocess
        path = None
        cwd = None
        close_fds = None
        shell = None

        if os.name is 'posix':
            path = sc_dir + sc_exe
            close_fds = True
            shell = False
        else:
            path = sc_exe
            cwd = sc_dir
            close_fds = False
            shell = True

        SuperColliderProcess.sclang_process = subprocess.Popen(
            args = [path, '-i', 'sublime'],
            cwd = cwd,
            bufsize = 0,
            stdin = subprocess.PIPE,
            stdout = subprocess.PIPE,
            stderr = subprocess.STDOUT,
            close_fds = close_fds,
            shell = shell
        )

        # create post window temp file
        SuperColliderProcess.open_post_view()

        # create post window update queue and thread
        def enqueue_output(input, queue):
            for line in iter(input.readline, b''):
                queue.put(line.decode('utf-8'))
            input.close()
            if SuperColliderProcess.has_post_view():
                SuperColliderProcess.deactivate_post_view('SublimeText: sclang terminated!\n')

        SuperColliderProcess.sclang_queue = Queue()
        SuperColliderProcess.sclang_thread = threading.Thread(
            target = enqueue_output,
            args = (
                SuperColliderProcess.sclang_process.stdout,
                SuperColliderProcess.sclang_queue
            )
        )

        SuperColliderProcess.sclang_thread.daemon = True # dies with the program
        SuperColliderProcess.sclang_thread.start()
        sublime.status_message("Starting SuperCollider")

    def stop():
        if SuperColliderProcess.is_alive():
            SuperColliderProcess.execute("0.exit;")
        else:
            sublime.status_message("sclang not running")

    def is_alive():
        return (SuperColliderProcess.sclang_thread is not None and
                SuperColliderProcess.sclang_thread.isAlive())

    def execute(cmd):
        if SuperColliderProcess.sclang_process is None:
            sublime.status_message("sclang not running")
            return

        SuperColliderProcess.sclang_process.stdin.write(bytes(cmd + '\x0c', 'utf-8'))
        SuperColliderProcess.sclang_process.stdin.flush()

    def has_post_view():
        return SuperColliderProcess.post_view is not None

    def post_view_id():
        if SuperColliderProcess.has_post_view():
            return SuperColliderProcess.post_view.id()
        else:
            return None

    def update_post_view():
        if SuperColliderProcess.is_alive() and SuperColliderProcess.has_post_view():
            try:
                line = SuperColliderProcess.sclang_queue.get_nowait()
            except Empty:
                pass
            else:
                SuperColliderProcess.post_view.run_command('super_collider_update_post_view', {
                    'content': line,
                    'max_lines': SuperColliderProcess.post_view_max_lines
                })

        sublime.set_timeout(SuperColliderProcess.update_post_view, 20)

    def cache_post_view(content):
        SuperColliderProcess.post_view_cache = content

    def deactivate_post_view(msg):
        if SuperColliderProcess.has_post_view():
            SuperColliderProcess.post_view.run_command('super_collider_update_post_view', {
                'content': msg,
                'force_scroll': True
            })
            if SuperColliderProcess.post_view is not None:
                SuperColliderProcess.post_view.set_name(SuperColliderProcess.post_view_name + ' - Inactive')
            SuperColliderProcess.post_view = None

    def open_post_view():
        if len(sublime.windows()) is 0:
            sublime.run_command('new_window')

        old_view = None

        if SuperColliderProcess.has_post_view():
            old_view = SuperColliderProcess.post_view
            # switch to the post window if it is in the current window
            for window in sublime.windows():
                if window.views().count(old_view):
                    window.focus_view(old_view)
                    return
                else:
                    # cache old post view contents
                    content = old_view.substr(sublime.Region(0, old_view.size()))
                    SuperColliderProcess.cache_post_view(content)

        window = sublime.active_window()

        # create new post view
        post_view = window.new_file()
        post_view.set_name(SuperColliderProcess.post_view_name)
        post_view.set_scratch(True)
        post_view.settings().set('rulers', 0)
        post_view.settings().set('line_numbers', False)

        # update the view with previoius view content if possible
        if SuperColliderProcess.post_view_cache is not None:
            post_view.run_command('super_collider_update_post_view', {
                'content': SuperColliderProcess.post_view_cache,
                'max_lines': SuperColliderProcess.post_view_max_lines,
                'force_scroll': True
            })
            SuperColliderProcess.post_view_cache = None

        # deactivate old view
        if old_view is not None:
            SuperColliderProcess.deactivate_post_view('Sublime Text: This post window now deactivated!\n')

        # update post view to newly created
        SuperColliderProcess.post_view = post_view
        SuperColliderProcess.update_post_view()

class SuperColliderStartCommand(sublime_plugin.ApplicationCommand):
    def run(self):
        SuperColliderProcess.start()

class SuperColliderStopCommand(sublime_plugin.ApplicationCommand):
    def run(self):
        SuperColliderProcess.stop()

class SuperColliderUpdatePostViewCommand(sublime_plugin.TextCommand):
    def run(self, edit, content, max_lines=-1, force_scroll=False):
        # insert text
        self.view.insert(edit, self.view.size(), content)

        # erase overspill
        all_lines = self.view.lines(sublime.Region(0, self.view.size()))
        total_lines = len(all_lines)
        if total_lines > max_lines and max_lines >= 1:
            end = all_lines[total_lines - max_lines].b + 1
            self.view.erase(edit, sublime.Region(0, end))

        # scroll
        if force_scroll or view_is_at_bottom(self.view):
            self.view.show(self.view.size())

class SuperColliderOpenPostViewCommand(sublime_plugin.ApplicationCommand):
    def run(self):
        SuperColliderProcess.open_post_view()

class SuperColliderListener(sublime_plugin.EventListener):
    def on_close(self, view):
        if view.id() is SuperColliderProcess.post_view_id():
            content = view.substr(sublime.Region(0, view.size()))
            SuperColliderProcess.cache_post_view(content)

class SuperColliderSendCommand(sublime_plugin.TextCommand):
    def run(self, edit):
        for sel in self.view.sel():
            cmd = None
            # "selection" is a single point
            if sel.a == sel.b:
                cmd = self.view.substr(self.view.line(sel))
            else:
                cmd = self.view.substr(sel)

            SuperColliderProcess.execute(cmd)

class SuperColliderLoop(sublime_plugin.ApplicationCommand):
    def run(self):
        SuperColliderProcess.execute("{inf.do{|x| x.postln; 0.1.wait; }}.fork")

class SuperColliderTest(sublime_plugin.ApplicationCommand):
    def run(self, count):
        SuperColliderProcess.execute(str(count) + ".do {|i| i.postln; };")

# TODO if re-open last tab and context SC, then open new post window
# TODO option on where to open post window: new tab, new group, new window, terminal
# TODO refocus on original window on open_post_view
# TODO re-write with plugin_loaded and process instance?<|endoftext|>


<commit_before>import argparse
import asyncio
import csv
import json
import logging
import pandas as pd
import requests
import sys
import time

from datetime import datetime
from pandas.io.json import json_normalize
from requests.packages.urllib3.exceptions import ReadTimeoutError
from sqlalchemy import func

from dataactcore.config import CONFIG_BROKER
from dataactcore.interfaces.db import GlobalDB
from dataactcore.logging import configure_logging
from dataactcore.models.domainModels import Office, SubTierAgency

from dataactvalidator.health_check import create_app
from dataactvalidator.filestreaming.csv_selection import write_query_to_file

logger = logging.getLogger(__name__)
logging.getLogger("requests").setLevel(logging.WARNING)

API_URL = CONFIG_BROKER['sam']['federal_hierarchy_api_url'].format(CONFIG_BROKER['sam']['federal_hierarchy_api_key'])
REQUESTS_AT_ONCE = 10

def pull_offices(sess, filename, update_db, pull_all, updated_date_from, export_office):
    """ Pull Office data from the Federal Hierarchy API and update the DB, return it as a file, or both.

        Args:
            sess: Current DB session.
            filename: Name of the file to be generated with the API data. If None, no file will be created.
            update_db: Boolean; update the DB tables with the new data from the API.
            pull_all: Boolean; pull all historical data, instead of just the latest.
            updated_date_from: Date to pull data from. Defaults to the date of the most recently updated Office.
    """
    logger.info('Starting feed: %s', API_URL.replace(CONFIG_BROKER['sam']['federal_hierarchy_api_key'], "[API_KEY]"))
    top_sub_levels = ["1", "2"]
    office_levels = ["3", "4", "5", "6", "7"]
    levels = top_sub_levels + office_levels if filename else office_levels

    if filename:
        logger.info("Creating a file ({}) with the data from this pull".format(filename))
        # Write headers to file
        file_headers = [
            "fhorgid", "fhorgname", "fhorgtype", "description", "level", "status", "region", "categoryid",
            "effectivestartdate", "effectiveenddate", "createdby", "createddate", "updatedby", "lastupdateddate",
            "fhdeptindagencyorgid", "fhagencyorgname", "agencycode", "oldfpdsofficecode", "aacofficecode",
            "cgaclist_0_cgac", "fhorgofficetypelist_0_officetype", "fhorgofficetypelist_0_officetypestartdate",
            "fhorgofficetypelist_0_officetypeenddate", "fhorgofficetypelist_1_officetype",
            "fhorgofficetypelist_1_officetypestartdate", "fhorgofficetypelist_1_officetypeenddate",
            "fhorgofficetypelist_2_officetype", "fhorgofficetypelist_2_officetypestartdate",
            "fhorgofficetypelist_2_officetypeenddate", "fhorgaddresslist_0_city", "fhorgaddresslist_0_state",
            "fhorgaddresslist_0_country_code", "fhorgaddresslist_0_addresstype", "fhorgnamehistory_0_fhorgname",
            "fhorgnamehistory_0_effectivedate", "fhorgparenthistory_0_fhfullparentpathid",
            "fhorgparenthistory_0_fhfullparentpathname", "fhorgparenthistory_0_effectivedate", "links_0_href",
            "links_0_rel", "links_1_href", "links_1_rel", "links_2_href", "links_2_rel"]
        with open(filename, 'w+') as f:
            csv_writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
            csv_writer.writerow(file_headers)

    empty_pull_count = 0
    for level in levels:
        # Create URL with the level parameter
        url_with_params = "{}&level={}".format(API_URL, level)

        # Add updateddatefrom parameter to the URL
        if not pull_all:
            url_with_params += "&updateddatefrom={}".format(updated_date_from)

        # Retrieve the total count of expected records for this pull
        total_expected_records = json.loads(requests.get(url_with_params, timeout=60).text)['totalrecords']
        logger.info('{} level-{} record(s) expected'.format(str(total_expected_records), str(level)))
        if total_expected_records == 0:
            empty_pull_count += 1
            continue

        limit = 100
        entries_processed = 0
        while True:
            async def _fed_hierarchy_async_get(entries_already_processed):
                response_list = []
                loop = asyncio.get_event_loop()
                futures = [
                    loop.run_in_executor(
                        None,
                        get_with_exception_hand,
                        "{}&limit={}&offset={}".format(url_with_params, str(limit),
                                                       str(entries_already_processed + (start_offset * limit)))
                    )
                    for start_offset in range(REQUESTS_AT_ONCE)
                ]
                for response in await asyncio.gather(*futures):
                    response_list.append(response.text)
                    pass
                return response_list
            # End async get requests def

            # Retrieve limit*REQUESTS_AT_ONCE records from the API
            loop = asyncio.get_event_loop()
            full_response = loop.run_until_complete(_fed_hierarchy_async_get(entries_processed))

            # Create an object with all the data from the API
            dataframe = pd.DataFrame()
            offices = {}
            start = entries_processed + 1
            for next_resp in full_response:
                response_dict = json.loads(next_resp)

                for org in response_dict.get('orglist', []):
                    entries_processed += 1

                    # Add to the file data structure
                    if filename:
                        row = json_normalize(flatten_json(org))
                        dataframe = dataframe.append(row)

                    # Don't process the top_sub_levels, but store them in the fed hierarchy export
                    if level in top_sub_levels:
                        continue

                    # Add to the list of DB objects
                    if update_db:
                        agency_code = get_normalized_agency_code(org.get('cgaclist', [{'cgac': None}])[0]['cgac'],
                                                                 org.get('agencycode'))
                        # TEMPORARILY REPLACE Navy, Army, AND Air Force WITH DOD
                        if agency_code in ['017', '021', '057']:
                            agency_code = '097'
                        if not org.get('aacofficecode') or not org.get('agencycode') or not agency_code:
                            # Item from Fed Hierarchy is missing necessary data, ignore it
                            continue

                        new_office = Office(office_code=org.get('aacofficecode'), office_name=org.get('fhorgname'),
                                            sub_tier_code=org.get('agencycode'), agency_code=agency_code,
                                            funding_office=False, contracting_office=False, grant_office=False)

                        for off_type in org.get('fhorgofficetypelist', []):
                            office_type = off_type['officetype'].lower()
                            if office_type == 'financial assistance':
                                office_type = 'grant'
                            if office_type in ['contracting', 'funding', 'grant']:
                                setattr(new_office, office_type + '_office', True)

                        offices[org.get('aacofficecode')] = new_office

            if filename and len(dataframe.index) > 0:
                # Ensure headers are handled correctly
                for header in list(dataframe.columns.values):
                    if header not in file_headers:
                        file_headers.append(header)
                        logger.info("Headers missing column: %s", header)

                # Write to file
                with open(filename, 'a') as f:
                    dataframe.to_csv(f, index=False, header=False, columns=file_headers)

            if update_db:
                office_codes = set(offices.keys())
                sess.query(Office).filter(Office.office_code.in_(office_codes)).delete(synchronize_session=False)
                sess.add_all(offices.values())

            logger.info("Processed rows %s-%s", start, entries_processed)
            if entries_processed == total_expected_records:
                # Feed has finished
                break

            if entries_processed > total_expected_records:
                # We have somehow retrieved more records than existed at the beginning of the pull
                logger.error("Total expected records: {}, Number of records retrieved: {}".format(
                    total_expected_records, entries_processed))
                sys.exit(2)

    if update_db:
        sess.commit()

    if export_office:
        logger.info("Creating a file ({}) with the data from the database".format(export_office))
        all_offices = sess.query(Office)
        write_query_to_file(sess, all_offices, export_office, generate_headers=True)

    if empty_pull_count == len(levels):
        logger.error("No records retrieved from the Federal Hierarchy API")
        sys.exit(3)

    logger.info("Finished")

def flatten_json(json_obj):
    """ Flatten a JSON object into a single row.

        Args:
            json_obj: JSON object to flatten

        Returns:
            Single row of values from the json_obj JSON
    """
    out = {}

    def _flatten(list_item, name=''):
        if type(list_item) is dict:
            for item in list_item:
                _flatten(list_item[item], name + item + '_')
        elif type(list_item) is list:
            count = 0
            for item in list_item:
                _flatten(item, name + str(count) + '_')
                count += 1
        else:
            out[name[:-1]] = list_item

    _flatten(json_obj)
    return out

def get_normalized_agency_code(agency_code, subtier_code):
    """ Handle the specific set of FREC agencies by matching the Office's SubtierAgency to its FREC

        Args:
            agency_code: CGAC agency code given to us by the Fed Hierarchy API
            subtier_code: SubtierAgency code under the selected CGAC agency. May map to a FREC instead

        Returns:
            FREC or CGAC agency code, depending on whether the speicified CGAC has been replaced with FREC
    """
    sess = GlobalDB.db().session
    if agency_code in ['011', '016', '352', '537', '033']:
        st_agency = sess.query(SubTierAgency).filter(SubTierAgency.sub_tier_agency_code == subtier_code).one_or_none()
        if st_agency:
            agency_code = st_agency.frec.frec_code
        else:
            agency_code = None

    return agency_code

def get_with_exception_hand(url_string):
    """ Retrieve data from API, allow for multiple retries and timeouts

        Args:
            url_string: URL to make the request to

        Returns:
            API response from the URL
    """
    exception_retries = -1
    retry_sleep_times = [5, 30, 60, 180, 300, 360, 420, 480, 540, 600]
    request_timeout = 60

    while exception_retries < len(retry_sleep_times):
        try:
            resp = requests.get(url_string, timeout=request_timeout)
            break
        except (ConnectionResetError, ReadTimeoutError, requests.exceptions.ConnectionError,
                requests.exceptions.ReadTimeout) as e:
            exception_retries += 1
            request_timeout += 60
            if exception_retries < len(retry_sleep_times):
                logger.info('Connection exception. Sleeping {}s and then retrying with a max wait of {}s...'
                            .format(retry_sleep_times[exception_retries], request_timeout))
                time.sleep(retry_sleep_times[exception_retries])
            else:
                logger.error('Connection to Federal Hierarchy API lost, maximum retry attempts exceeded.')
                sys.exit(2)
    return resp

def main():
    parser = argparse.ArgumentParser(description='Pull data from the Federal Hierarchy API.')
    parser.add_argument('-a', '--all', help='Clear out the database and get historical data', action='store_true')
    parser.add_argument('-f', '--filename', help='Generate a local CSV file from the data.', nargs=1, type=str)
    parser.add_argument('-o', '--export_office', help='Export the current office table. '
                                                      'Please provide the file name/path.', nargs=1, type=str)
    parser.add_argument('-d', '--pull_date', help='Date from which to start the pull', nargs=1, type=str)
    parser.add_argument('-i', '--ignore_db', help='Do not update the DB tables', action='store_true')
    args = parser.parse_args()

    if args.all and args.pull_date:
        logger.error("The -a and -d flags conflict, cannot use both at once.")
        sys.exit(1)

    # Handle the pull_date parameter
    updated_date_from = None
    if args.pull_date:
        try:
            updated_date_from = args.pull_date[0]
            datetime.strptime(updated_date_from, "%Y-%m-%d")
        except ValueError as e:
            logger.error("The date given to the -d flag was not parseable.")
            sys.exit(1)

    # Get or create the start date
    sess = GlobalDB.db().session
    if not args.all and not updated_date_from:
        last_pull_date = sess.query(func.max(Office.updated_at)).one_or_none()
        if not last_pull_date:
            logger.error("The -a or -d flag must be set when there are no Offices present in the database.")
            sys.exit(1)
        updated_date_from = last_pull_date[0].date()

    # Handle the filename parameter
    filename = args.filename[0] if args.filename else None

    # Handle the export office parameter
    export_office = args.export_office[0] if args.export_office else None

    # Handle a complete data reload
    if args.all and not args.ignore_db:
        logger.info("Emptying out the Office table for a complete reload.")
        sess.execute('''TRUNCATE TABLE office RESTART IDENTITY''')

    try:
        pull_offices(sess, filename, not args.ignore_db, args.all, updated_date_from, export_office)
    except Exception as e:
        logger.error(str(e))
        sys.exit(1)

if __name__ == '__main__':
    with create_app().app_context():
        configure_logging()
        main()
<commit_msg>Update script to use the name<commit_after>import argparse
import asyncio
import csv
import json
import logging
import pandas as pd
import requests
import sys
import time

from datetime import datetime
from pandas.io.json import json_normalize
from requests.packages.urllib3.exceptions import ReadTimeoutError
from sqlalchemy import func

from dataactcore.config import CONFIG_BROKER
from dataactcore.interfaces.db import GlobalDB
from dataactcore.logging import configure_logging
from dataactcore.models.domainModels import Office, SubTierAgency

from dataactvalidator.health_check import create_app
from dataactvalidator.filestreaming.csv_selection import write_query_to_file

logger = logging.getLogger(__name__)
logging.getLogger("requests").setLevel(logging.WARNING)

API_URL = CONFIG_BROKER['sam']['federal_hierarchy_api_url'].format(CONFIG_BROKER['sam']['federal_hierarchy_api_key'])
REQUESTS_AT_ONCE = 10

def pull_offices(sess, filename, update_db, pull_all, updated_date_from, export_office):
    """ Pull Office data from the Federal Hierarchy API and update the DB, return it as a file, or both.

        Args:
            sess: Current DB session.
            filename: Name of the file to be generated with the API data. If None, no file will be created.
            update_db: Boolean; update the DB tables with the new data from the API.
            pull_all: Boolean; pull all historical data, instead of just the latest.
            updated_date_from: Date to pull data from. Defaults to the date of the most recently updated Office.
    """
    logger.info('Starting feed: %s', API_URL.replace(CONFIG_BROKER['sam']['federal_hierarchy_api_key'], "[API_KEY]"))
    top_sub_levels = ["1", "2"]
    office_levels = ["3", "4", "5", "6", "7"]
    levels = top_sub_levels + office_levels if filename else office_levels

    if filename:
        logger.info("Creating a file ({}) with the data from this pull".format(filename))
        # Write headers to file
        file_headers = [
            "fhorgid", "fhorgname", "fhorgtype", "description", "level", "status", "region", "categoryid",
            "effectivestartdate", "effectiveenddate", "createdby", "createddate", "updatedby", "lastupdateddate",
            "fhdeptindagencyorgid", "fhagencyorgname", "agencycode", "oldfpdsofficecode", "aacofficecode",
            "cgaclist_0_cgac", "fhorgofficetypelist_0_officetype", "fhorgofficetypelist_0_officetypestartdate",
            "fhorgofficetypelist_0_officetypeenddate", "fhorgofficetypelist_1_officetype",
            "fhorgofficetypelist_1_officetypestartdate", "fhorgofficetypelist_1_officetypeenddate",
            "fhorgofficetypelist_2_officetype", "fhorgofficetypelist_2_officetypestartdate",
            "fhorgofficetypelist_2_officetypeenddate", "fhorgaddresslist_0_city", "fhorgaddresslist_0_state",
            "fhorgaddresslist_0_country_code", "fhorgaddresslist_0_addresstype", "fhorgnamehistory_0_fhorgname",
            "fhorgnamehistory_0_effectivedate", "fhorgparenthistory_0_fhfullparentpathid",
            "fhorgparenthistory_0_fhfullparentpathname", "fhorgparenthistory_0_effectivedate", "links_0_href",
            "links_0_rel", "links_1_href", "links_1_rel", "links_2_href", "links_2_rel"]
        with open(filename, 'w+') as f:
            csv_writer = csv.writer(f, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL)
            csv_writer.writerow(file_headers)

    empty_pull_count = 0
    for level in levels:
        # Create URL with the level parameter
        url_with_params = "{}&level={}".format(API_URL, level)

        # Add updateddatefrom parameter to the URL
        if not pull_all:
            url_with_params += "&updateddatefrom={}".format(updated_date_from)

        # Retrieve the total count of expected records for this pull
        total_expected_records = json.loads(requests.get(url_with_params, timeout=60).text)['totalrecords']
        logger.info('{} level-{} record(s) expected'.format(str(total_expected_records), str(level)))
        if total_expected_records == 0:
            empty_pull_count += 1
            continue

        limit = 100
        entries_processed = 0
        while True:
            async def _fed_hierarchy_async_get(entries_already_processed):
                response_list = []
                loop = asyncio.get_event_loop()
                futures = [
                    loop.run_in_executor(
                        None,
                        get_with_exception_hand,
                        "{}&limit={}&offset={}".format(url_with_params, str(limit),
                                                       str(entries_already_processed + (start_offset * limit)))
                    )
                    for start_offset in range(REQUESTS_AT_ONCE)
                ]
                for response in await asyncio.gather(*futures):
                    response_list.append(response.text)
                    pass
                return response_list
            # End async get requests def

            # Retrieve limit*REQUESTS_AT_ONCE records from the API
            loop = asyncio.get_event_loop()
            full_response = loop.run_until_complete(_fed_hierarchy_async_get(entries_processed))

            # Create an object with all the data from the API
            dataframe = pd.DataFrame()
            offices = {}
            start = entries_processed + 1
            for next_resp in full_response:
                response_dict = json.loads(next_resp)

                for org in response_dict.get('orglist', []):
                    entries_processed += 1

                    # Add to the file data structure
                    if filename:
                        row = json_normalize(flatten_json(org))
                        dataframe = dataframe.append(row)

                    # Don't process the top_sub_levels, but store them in the fed hierarchy export
                    if level in top_sub_levels:
                        continue

                    # Add to the list of DB objects
                    if update_db:
                        agency_code = get_normalized_agency_code(org.get('cgaclist', [{'cgac': None}])[0]['cgac'],
                                                                 org.get('agencycode'))
                        # TEMPORARILY REPLACE Navy, Army, AND Air Force WITH DOD
                        if agency_code in ['017', '021', '057']:
                            agency_code = '097'
                        if not org.get('aacofficecode') or not org.get('agencycode') or not agency_code:
                            # Item from Fed Hierarchy is missing necessary data, ignore it
                            continue

                        new_office = Office(office_code=org.get('aacofficecode'), office_name=org.get('fhorgname'),
                                            sub_tier_code=org.get('agencycode'), agency_code=agency_code,
                                            funding_office=False, contracting_office=False,
                                            financial_assistance_office=False)

                        for off_type in org.get('fhorgofficetypelist', []):
                            office_type = off_type['officetype'].lower().replace(" ", "_")
                            if office_type in ['contracting', 'funding', 'financial_assistance']:
                                setattr(new_office, office_type + '_office', True)

                        offices[org.get('aacofficecode')] = new_office

            if filename and len(dataframe.index) > 0:
                # Ensure headers are handled correctly
                for header in list(dataframe.columns.values):
                    if header not in file_headers:
                        file_headers.append(header)
                        logger.info("Headers missing column: %s", header)

                # Write to file
                with open(filename, 'a') as f:
                    dataframe.to_csv(f, index=False, header=False, columns=file_headers)

            if update_db:
                office_codes = set(offices.keys())
                sess.query(Office).filter(Office.office_code.in_(office_codes)).delete(synchronize_session=False)
                sess.add_all(offices.values())

            logger.info("Processed rows %s-%s", start, entries_processed)
            if entries_processed == total_expected_records:
                # Feed has finished
                break

            if entries_processed > total_expected_records:
                # We have somehow retrieved more records than existed at the beginning of the pull
                logger.error("Total expected records: {}, Number of records retrieved: {}".format(
                    total_expected_records, entries_processed))
                sys.exit(2)

    if update_db:
        sess.commit()

    if export_office:
        logger.info("Creating a file ({}) with the data from the database".format(export_office))
        all_offices = sess.query(Office)
        write_query_to_file(sess, all_offices, export_office, generate_headers=True)

    if empty_pull_count == len(levels):
        logger.error("No records retrieved from the Federal Hierarchy API")
        sys.exit(3)

    logger.info("Finished")

def flatten_json(json_obj):
    """ Flatten a JSON object into a single row.

        Args:
            json_obj: JSON object to flatten

        Returns:
            Single row of values from the json_obj JSON
    """
    out = {}

    def _flatten(list_item, name=''):
        if type(list_item) is dict:
            for item in list_item:
                _flatten(list_item[item], name + item + '_')
        elif type(list_item) is list:
            count = 0
            for item in list_item:
                _flatten(item, name + str(count) + '_')
                count += 1
        else:
            out[name[:-1]] = list_item

    _flatten(json_obj)
    return out

def get_normalized_agency_code(agency_code, subtier_code):
    """ Handle the specific set of FREC agencies by matching the Office's SubtierAgency to its FREC

        Args:
            agency_code: CGAC agency code given to us by the Fed Hierarchy API
            subtier_code: SubtierAgency code under the selected CGAC agency. May map to a FREC instead

        Returns:
            FREC or CGAC agency code, depending on whether the speicified CGAC has been replaced with FREC
    """
    sess = GlobalDB.db().session
    if agency_code in ['011', '016', '352', '537', '033']:
        st_agency = sess.query(SubTierAgency).filter(SubTierAgency.sub_tier_agency_code == subtier_code).one_or_none()
        if st_agency:
            agency_code = st_agency.frec.frec_code
        else:
            agency_code = None

    return agency_code

def get_with_exception_hand(url_string):
    """ Retrieve data from API, allow for multiple retries and timeouts

        Args:
            url_string: URL to make the request to

        Returns:
            API response from the URL
    """
    exception_retries = -1
    retry_sleep_times = [5, 30, 60, 180, 300, 360, 420, 480, 540, 600]
    request_timeout = 60

    while exception_retries < len(retry_sleep_times):
        try:
            resp = requests.get(url_string, timeout=request_timeout)
            break
        except (ConnectionResetError, ReadTimeoutError, requests.exceptions.ConnectionError,
                requests.exceptions.ReadTimeout) as e:
            exception_retries += 1
            request_timeout += 60
            if exception_retries < len(retry_sleep_times):
                logger.info('Connection exception. Sleeping {}s and then retrying with a max wait of {}s...'
                            .format(retry_sleep_times[exception_retries], request_timeout))
                time.sleep(retry_sleep_times[exception_retries])
            else:
                logger.error('Connection to Federal Hierarchy API lost, maximum retry attempts exceeded.')
                sys.exit(2)
    return resp

def main():
    parser = argparse.ArgumentParser(description='Pull data from the Federal Hierarchy API.')
    parser.add_argument('-a', '--all', help='Clear out the database and get historical data', action='store_true')
    parser.add_argument('-f', '--filename', help='Generate a local CSV file from the data.', nargs=1, type=str)
    parser.add_argument('-o', '--export_office', help='Export the current office table. '
                                                      'Please provide the file name/path.', nargs=1, type=str)
    parser.add_argument('-d', '--pull_date', help='Date from which to start the pull', nargs=1, type=str)
    parser.add_argument('-i', '--ignore_db', help='Do not update the DB tables', action='store_true')
    args = parser.parse_args()

    if args.all and args.pull_date:
        logger.error("The -a and -d flags conflict, cannot use both at once.")
        sys.exit(1)

    # Handle the pull_date parameter
    updated_date_from = None
    if args.pull_date:
        try:
            updated_date_from = args.pull_date[0]
            datetime.strptime(updated_date_from, "%Y-%m-%d")
        except ValueError as e:
            logger.error("The date given to the -d flag was not parseable.")
            sys.exit(1)

    # Get or create the start date
    sess = GlobalDB.db().session
    if not args.all and not updated_date_from:
        last_pull_date = sess.query(func.max(Office.updated_at)).one_or_none()
        if not last_pull_date:
            logger.error("The -a or -d flag must be set when there are no Offices present in the database.")
            sys.exit(1)
        updated_date_from = last_pull_date[0].date()

    # Handle the filename parameter
    filename = args.filename[0] if args.filename else None

    # Handle the export office parameter
    export_office = args.export_office[0] if args.export_office else None

    # Handle a complete data reload
    if args.all and not args.ignore_db:
        logger.info("Emptying out the Office table for a complete reload.")
        sess.execute('''TRUNCATE TABLE office RESTART IDENTITY''')

    try:
        pull_offices(sess, filename, not args.ignore_db, args.all, updated_date_from, export_office)
    except Exception as e:
        logger.error(str(e))
        sys.exit(1)

if __name__ == '__main__':
    with create_app().app_context():
        configure_logging()
        main()
<|endoftext|>


<commit_before>from isochrones.dartmouth import Dartmouth_Isochrone
from isochrones.starmodel import StarModel
from isochrones.observation import ObservationTree
import pandas as pd
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

f1 = open('evidence_bound.txt','w')
f2 = open('evidence_unassociated.txt','w')

for n in range(0,2,1):
    if n < 10:
        i = '0' + str(n)
    else:
        i = str(n)

    #same system case
    df = pd.read_csv('df_binary_test{}.csv'.format(i))

    dar = Dartmouth_Isochrone()
    t = ObservationTree.from_df(df, name='test{}'.format(i))
    t.define_models(dar)

    mod = StarModel(dar, obs=t)
    mod.fit_multinest(n_live_points=1000,
                        basename='test{}_bound'.format(i))
    evi = mod.evidence
    evi = str(evi)
    f1.write('case{}: '.format(i) + evi + '\n')

    #mod.corner(['mass_0_0','mass_0_1','distance_0','AV_0'])
    mod.corner_physical(props=['mass', 'distance', 'AV'])
    mod.corner_observed()

    #unassociated case
    dar = Dartmouth_Isochrone()
    t = ObservationTree.from_df(df, name='test{}'.format(i))
    t.define_models(dar, index = [0,1])

    mod = StarModel(dar, obs=t)
    mod.fit_multinest(n_live_points=1000,
                        basename='test{}_unassociated'.format(i))
    evi = mod.evidence
    evi = str(evi)
    f2.write('case{}: '.format(i) + evi + '\n')

    #mod.corner(['mass_0_0','mass_0_1','distance_0','distance_1','AV_0','AV_1'])
    mod.corner_physical(props=['mass', 'distance', 'AV'])
    mod.corner_observed()

f1.close()
f2.close()
<commit_msg>update<commit_after>import matplotlib
matplotlib.use('Agg')
from isochrones.dartmouth import Dartmouth_Isochrone
from isochrones.starmodel import StarModel
from isochrones.observation import ObservationTree
import pandas as pd
import matplotlib.pyplot as plt

f1 = open('evidence_bound.txt','w')
f2 = open('evidence_unassociated.txt','w')

for n in range(0,2,1):
    if n < 10:
        i = '0' + str(n)
    else:
        i = str(n)

    #same system case
    df = pd.read_csv('df_binary_test{}.csv'.format(i))

    dar = Dartmouth_Isochrone()
    t = ObservationTree.from_df(df, name='test{}'.format(i))
    t.define_models(dar)

    mod = StarModel(dar, obs=t)
    mod.fit_multinest(n_live_points=1000,
                        basename='test{}_bound'.format(i))
    evi = mod.evidence
    evi = str(evi)
    f1.write('case{}: '.format(i) + evi + '\n')

    #mod.corner(['mass_0_0','mass_0_1','distance_0','AV_0'])
    mod.corner_physical(props=['mass', 'distance', 'AV'])
    mod.corner_observed()

    #unassociated case
    dar = Dartmouth_Isochrone()
    t = ObservationTree.from_df(df, name='test{}'.format(i))
    t.define_models(dar, index = [0,1])

    mod = StarModel(dar, obs=t)
    mod.fit_multinest(n_live_points=1000,
                        basename='test{}_unassociated'.format(i))
    evi = mod.evidence
    evi = str(evi)
    f2.write('case{}: '.format(i) + evi + '\n')

    #mod.corner(['mass_0_0','mass_0_1','distance_0','distance_1','AV_0','AV_1'])
    mod.corner_physical(props=['mass', 'distance', 'AV'])
    mod.corner_observed()

f1.close()
f2.close()
<|endoftext|>


<commit_before>import cPickle

import numpy
from pylearn.algorithms.mcRBM import mcRBM, mcRBMTrainer
import pylearn.datasets.cifar10
import theano
from theano import tensor

def l2(X):
    return numpy.sqrt((X**2).sum())

def _default_rbm_alloc(n_I, n_K=256, n_J=100):
    return mcRBM.alloc(n_I, n_K, n_J)

def _default_trainer_alloc(rbm, train_batch, batchsize, l1_penalty, l1_penalty_start):
    return mcRBMTrainer.alloc(rbm, train_batch, batchsize, l1_penalty=l1_penalty,
            l1_penalty_start=l1_penalty_start,persistent_chains=persistent_chains)

def test_reproduce_ranzato_hinton_2010(dataset='MAR',
        n_train_iters=5000,
        rbm_alloc=_default_rbm_alloc,
        trainer_alloc=_default_trainer_alloc,
        lr_per_example=.075,
        l1_penalty=1e-3,
        l1_penalty_start=1000,
        persistent_chains=True,
        ):

    batchsize = 128
    ## specific to MAR dataset ##
    n_vis=105
    n_patches=10240
    epoch_size=n_patches

    tile = pylearn.dataset_ops.image_patches.save_filters_of_ranzato_hinton_2010

    batch_idx = tensor.iscalar()
    batch_range =batch_idx * batchsize + numpy.arange(batchsize)

    train_batch = pylearn.dataset_ops.image_patches.ranzato_hinton_2010_op(batch_range)

    imgs_fn = theano.function([batch_idx], outputs=train_batch)

    trainer = trainer_alloc(
            rbm_alloc(n_I=n_vis),
            train_batch,
            batchsize,
            initial_lr_per_example=lr_per_example,
            l1_penalty=l1_penalty,
            l1_penalty_start=l1_penalty_start,
            persistent_chains=persistent_chains)
    rbm=trainer.rbm

    if persistent_chains:
        grads = trainer.contrastive_grads()
        learn_fn = theano.function([batch_idx],
                outputs=[grads[0].norm(2), grads[0].norm(2), grads[1].norm(2)],
                updates=trainer.cd_updates())
    else:
        learn_fn = theano.function([batch_idx], outputs=[], updates=trainer.cd_updates())

    if persistent_chains:
        smplr = trainer.sampler
    else:
        smplr = trainer._last_cd1_sampler

    if dataset == 'cifar10patches8x8':
        cPickle.dump(
                pylearn.dataset_ops.cifar10.random_cifar_patches_pca(
                    n_vis, None, 'float32', n_patches, R, C,),
                open('test_mcRBM.pca.pkl','w'))

    print "Learning..."
    last_epoch = -1
    for jj in xrange(n_train_iters):
        epoch = jj*batchsize / epoch_size

        print_jj = epoch != last_epoch
        last_epoch = epoch

        if print_jj:
            tile(imgs_fn(jj), "imgs_%06i.png"%jj)
            if persistent_chains:
                tile(smplr.positions.get_value(borrow=True), "sample_%06i.png"%jj)
            tile(rbm.U.get_value(borrow=True).T, "U_%06i.png"%jj)
            tile(rbm.W.get_value(borrow=True).T, "W_%06i.png"%jj)

            print 'saving samples', jj, 'epoch', jj/(epoch_size/batchsize)

            print 'l2(U)', l2(rbm.U.get_value(borrow=True)),
            print 'l2(W)', l2(rbm.W.get_value(borrow=True)),
            print 'l1_penalty',
            try:
                print trainer.effective_l1_penalty.get_value()
            except:
                print trainer.effective_l1_penalty

            print 'U min max', rbm.U.get_value(borrow=True).min(), rbm.U.get_value(borrow=True).max(),
            print 'W min max', rbm.W.get_value(borrow=True).min(), rbm.W.get_value(borrow=True).max(),
            print 'a min max', rbm.a.get_value(borrow=True).min(), rbm.a.get_value(borrow=True).max(),
            print 'b min max', rbm.b.get_value(borrow=True).min(), rbm.b.get_value(borrow=True).max(),
            print 'c min max', rbm.c.get_value(borrow=True).min(), rbm.c.get_value(borrow=True).max()

            if persistent_chains:
                print 'parts min', smplr.positions.get_value(borrow=True).min(),
                print 'max',smplr.positions.get_value(borrow=True).max(),
            print 'HMC step', smplr.stepsize.get_value(),
            print 'arate', smplr.avg_acceptance_rate.get_value()

        l2_of_Ugrad = learn_fn(jj)

        if persistent_chains and print_jj:
            print 'l2(U_grad)', float(l2_of_Ugrad[0]),
            print 'l2(U_inc)', float(l2_of_Ugrad[1]),
            print 'l2(W_inc)', float(l2_of_Ugrad[2]),
            #print 'FE+', float(l2_of_Ugrad[2]),
            #print 'FE+[0]', float(l2_of_Ugrad[3]),
            #print 'FE+[1]', float(l2_of_Ugrad[4]),
            #print 'FE+[2]', float(l2_of_Ugrad[5]),
            #print 'FE+[3]', float(l2_of_Ugrad[6])

        if jj % 2000 == 0:
            print ''
            print 'Saving rbm...'
            cPickle.dump(rbm, open('mcRBM.rbm.%06i.pkl'%jj, 'w'), -1)
            if persistent_chains:
                print 'Saving sampler...'
                cPickle.dump(smplr, open('mcRBM.smplr.%06i.pkl'%jj, 'w'), -1)

    return rbm, smplr
<commit_msg>bugfix: fix missing variable.<commit_after>import cPickle

import numpy
from pylearn.algorithms.mcRBM import mcRBM, mcRBMTrainer
import pylearn.datasets.cifar10
import theano
from theano import tensor

def l2(X):
    return numpy.sqrt((X**2).sum())

def _default_rbm_alloc(n_I, n_K=256, n_J=100):
    return mcRBM.alloc(n_I, n_K, n_J)

def _default_trainer_alloc(rbm, train_batch, batchsize, initial_lr_per_example,
                           l1_penalty, l1_penalty_start, persistent_chains):
    return mcRBMTrainer.alloc(rbm, train_batch, batchsize, l1_penalty=l1_penalty,
            l1_penalty_start=l1_penalty_start,persistent_chains=persistent_chains)

def test_reproduce_ranzato_hinton_2010(dataset='MAR',
        n_train_iters=5000,
        rbm_alloc=_default_rbm_alloc,
        trainer_alloc=_default_trainer_alloc,
        lr_per_example=.075,
        l1_penalty=1e-3,
        l1_penalty_start=1000,
        persistent_chains=True,
        ):

    batchsize = 128
    ## specific to MAR dataset ##
    n_vis=105
    n_patches=10240
    epoch_size=n_patches

    tile = pylearn.dataset_ops.image_patches.save_filters_of_ranzato_hinton_2010

    batch_idx = tensor.iscalar()
    batch_range =batch_idx * batchsize + numpy.arange(batchsize)

    train_batch = pylearn.dataset_ops.image_patches.ranzato_hinton_2010_op(batch_range)

    imgs_fn = theano.function([batch_idx], outputs=train_batch)

    trainer = trainer_alloc(
            rbm_alloc(n_I=n_vis),
            train_batch,
            batchsize,
            initial_lr_per_example=lr_per_example,
            l1_penalty=l1_penalty,
            l1_penalty_start=l1_penalty_start,
            persistent_chains=persistent_chains)
    rbm=trainer.rbm

    if persistent_chains:
        grads = trainer.contrastive_grads()
        learn_fn = theano.function([batch_idx],
                outputs=[grads[0].norm(2), grads[0].norm(2), grads[1].norm(2)],
                updates=trainer.cd_updates())
    else:
        learn_fn = theano.function([batch_idx], outputs=[], updates=trainer.cd_updates())

    if persistent_chains:
        smplr = trainer.sampler
    else:
        smplr = trainer._last_cd1_sampler

    if dataset == 'cifar10patches8x8':
        cPickle.dump(
                pylearn.dataset_ops.cifar10.random_cifar_patches_pca(
                    n_vis, None, 'float32', n_patches, R, C,),
                open('test_mcRBM.pca.pkl','w'))

    print "Learning..."
    last_epoch = -1
    for jj in xrange(n_train_iters):
        epoch = jj*batchsize / epoch_size

        print_jj = epoch != last_epoch
        last_epoch = epoch

        if print_jj:
            tile(imgs_fn(jj), "imgs_%06i.png"%jj)
            if persistent_chains:
                tile(smplr.positions.get_value(borrow=True), "sample_%06i.png"%jj)
            tile(rbm.U.get_value(borrow=True).T, "U_%06i.png"%jj)
            tile(rbm.W.get_value(borrow=True).T, "W_%06i.png"%jj)

            print 'saving samples', jj, 'epoch', jj/(epoch_size/batchsize)

            print 'l2(U)', l2(rbm.U.get_value(borrow=True)),
            print 'l2(W)', l2(rbm.W.get_value(borrow=True)),
            print 'l1_penalty',
            try:
                print trainer.effective_l1_penalty.get_value()
            except:
                print trainer.effective_l1_penalty

            print 'U min max', rbm.U.get_value(borrow=True).min(), rbm.U.get_value(borrow=True).max(),
            print 'W min max', rbm.W.get_value(borrow=True).min(), rbm.W.get_value(borrow=True).max(),
            print 'a min max', rbm.a.get_value(borrow=True).min(), rbm.a.get_value(borrow=True).max(),
            print 'b min max', rbm.b.get_value(borrow=True).min(), rbm.b.get_value(borrow=True).max(),
            print 'c min max', rbm.c.get_value(borrow=True).min(), rbm.c.get_value(borrow=True).max()

            if persistent_chains:
                print 'parts min', smplr.positions.get_value(borrow=True).min(),
                print 'max',smplr.positions.get_value(borrow=True).max(),
            print 'HMC step', smplr.stepsize.get_value(),
            print 'arate', smplr.avg_acceptance_rate.get_value()

        l2_of_Ugrad = learn_fn(jj)

        if persistent_chains and print_jj:
            print 'l2(U_grad)', float(l2_of_Ugrad[0]),
            print 'l2(U_inc)', float(l2_of_Ugrad[1]),
            print 'l2(W_inc)', float(l2_of_Ugrad[2]),
            #print 'FE+', float(l2_of_Ugrad[2]),
            #print 'FE+[0]', float(l2_of_Ugrad[3]),
            #print 'FE+[1]', float(l2_of_Ugrad[4]),
            #print 'FE+[2]', float(l2_of_Ugrad[5]),
            #print 'FE+[3]', float(l2_of_Ugrad[6])

        if jj % 2000 == 0:
            print ''
            print 'Saving rbm...'
            cPickle.dump(rbm, open('mcRBM.rbm.%06i.pkl'%jj, 'w'), -1)
            if persistent_chains:
                print 'Saving sampler...'
                cPickle.dump(smplr, open('mcRBM.smplr.%06i.pkl'%jj, 'w'), -1)

    return rbm, smplr
<|endoftext|>


<commit_before># Generated by Django 3.2.13 on 2022-05-02 11:06

from django.db import migrations
from django.db.models import F
import uuid
import datetime
import pytz

BATCH_SIZE = 10000

def set_order_discount_token_and_created_at(apps, _schema_editor):
    OrderDiscount = apps.get_model("discount", "OrderDiscount")
    Order = apps.get_model("order", "Order")

    queryset = OrderDiscount.objects.filter(token__isnull=True)
    for batch_pks in queryset_in_batches(queryset):
        discounts = OrderDiscount.objects.filter(pk__in=batch_pks)
        order_in_bulk = Order.objects.in_bulk(
            discounts.values_list("order_id", flat=True)
        )
        for discount in discounts:
            order = order_in_bulk.get(discount.order_id)
            discount.created_at = (
                order.created_at.replace(second=discount.pk)
                if order
                else datetime.datetime.now(pytz.UTC)
            )
            discount.token = uuid.uuid4()

        OrderDiscount.objects.bulk_update(discounts, ["token", "created_at"])

def queryset_in_batches(queryset):
    """Slice a queryset into batches.

    Input queryset should be sorted be pk.
    """
    start_pk = 0

    while True:
        qs = queryset.order_by("pk").filter(pk__gt=start_pk)[:BATCH_SIZE]
        pks = list(qs.values_list("pk", flat=True))

        if not pks:
            break

        yield pks

        start_pk = pks[-1]

def set_order_discount_old_id(apps, schema_editor):
    OrderDiscount = apps.get_model("discount", "OrderDiscount")
    OrderDiscount.objects.all().update(old_id=F("id"))

class Migration(migrations.Migration):

    dependencies = [
        ("discount", "0040_orderdiscount_token_old_id_created_at"),
    ]

    operations = [
        migrations.RunPython(
            set_order_discount_token_and_created_at,
            migrations.RunPython.noop,
        ),
        migrations.RunPython(
            set_order_discount_old_id, reverse_code=migrations.RunPython.noop
        ),
    ]
<commit_msg>Fix settings seconds in migrations (#9885)<commit_after># Generated by Django 3.2.13 on 2022-05-02 11:06

from django.db import migrations
from django.db.models import F
import uuid
import datetime
import pytz

BATCH_SIZE = 10000

def set_order_discount_token_and_created_at(apps, _schema_editor):
    OrderDiscount = apps.get_model("discount", "OrderDiscount")
    Order = apps.get_model("order", "Order")

    queryset = OrderDiscount.objects.filter(token__isnull=True)
    for batch_pks in queryset_in_batches(queryset):
        discounts = OrderDiscount.objects.filter(pk__in=batch_pks)
        order_in_bulk = Order.objects.in_bulk(
            discounts.values_list("order_id", flat=True)
        )
        for discount in discounts:
            order = order_in_bulk.get(discount.order_id)
            discount.created_at = (
                order.created_at if order else datetime.datetime.now(pytz.UTC)
            )
            discount.token = uuid.uuid4()

        OrderDiscount.objects.bulk_update(discounts, ["token", "created_at"])

def queryset_in_batches(queryset):
    """Slice a queryset into batches.

    Input queryset should be sorted be pk.
    """
    start_pk = 0

    while True:
        qs = queryset.order_by("pk").filter(pk__gt=start_pk)[:BATCH_SIZE]
        pks = list(qs.values_list("pk", flat=True))

        if not pks:
            break

        yield pks

        start_pk = pks[-1]

def set_order_discount_old_id(apps, schema_editor):
    OrderDiscount = apps.get_model("discount", "OrderDiscount")
    OrderDiscount.objects.all().update(old_id=F("id"))

class Migration(migrations.Migration):

    dependencies = [
        ("discount", "0040_orderdiscount_token_old_id_created_at"),
    ]

    operations = [
        migrations.RunPython(
            set_order_discount_token_and_created_at,
            migrations.RunPython.noop,
        ),
        migrations.RunPython(
            set_order_discount_old_id, reverse_code=migrations.RunPython.noop
        ),
    ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    flask.ext.riaksessions
    ----------------------

    This module provides a Riak-backed session store for Flask sessions.

    :copyright: (c) 2013 Tim Gross
    :license: MIT

"""

__version_info__ = ('0', '1', '0')
__version__ = '.'.join(__version_info__)
__author__ = 'Tim Gross'
__license__ = 'MIT'
__copyright__ = '(c) 2013 Tim Gross'
__all__ = ['RiakSessions', 'RiakSessionInterface', 'RiakSession']

import base64
from datetime import datetime
import hmac
import hashlib
import json
import logging
import os

import riak
from werkzeug.datastructures import CallbackDict
from flask.sessions import SessionInterface, SessionMixin

# shut up about short parameter names
# pylint: disable=C0103
LOG = logging.getLogger(__name__)

def get_hmac(app, content):
    """
    Generates the hash message authentication code (HMAC) using the application
    configuration and the passed content. The content should include a salt.
    Takes the RIAK_SESSIONS_HASH_FUNCTION and SECRET_KEY app config.
    """
    hash_function = app.config['RIAK_SESSIONS_HASH_FUNCTION']
    secret = app.config['SECRET_KEY']
    return base64.b64encode(hmac.new(secret, content, hash_function).digest())

class Encoder(json.JSONEncoder):
    """
    Subclasses JSONEncoder to handle datetimes.
    """

    def default(self, obj):
        """
        Encodes datetime objects in annotated ISO-format:
        '/Date(2013-06-25T23:57:32.779968)/'
        """
        if isinstance(obj, datetime):
            return u'/Date({})/'.format(obj.isoformat())
        return super(Encoder, self).default(self, obj)

def date_hook(data):
    """
    Decodes strings that match the pattern created by the Endoder into datetime
    objects.
    """
    for key, val in data.items():
        if val.startswith('/Date('):
            data[key] = datetime.strptime(val.lstrip('/Date(').rstrip(')/'),
                                          "%Y-%m-%dT%H:%M:%S.%f")
    return data

class SessionValidationError(Exception):
    """
    Raised in the event a session id cannot be validated and should ignored.
    """
    pass

class RiakSession(CallbackDict, SessionMixin):
    """
    Implements a Flask session that can be HMAC-signed.
    """

    def __init__(self, initial=None, sid=None, new=False, expiry=None):
        def on_update(self):
            """
            Set the dirty-session flag.
            """
            self.modified = True

        CallbackDict.__init__(self, initial, on_update)
        self.sid = sid
        self.new = new
        self.modified = False
        self.expiry = expiry

class RiakSessionInterface(SessionInterface):
    """
    Implements a Flask session interface that uses Riak as the backing store
    and HMAC-signs the session cookie.
    """

    def __init__(self, app, client=None):
        if client is None:
            client = riak.RiakClient(port=app.config['RIAK_HTTP_PORT'])
        self.client = client

    def generate_sid(self, app, ip, user_agent):
        """
        Uses the IP and user-agent as the inputs to an HMAC to be used as the
        unique session ID.

        """
        sid = get_hmac(app, '{}.{}'.format(ip, base64.b64encode(user_agent)))
        return sid

    def put(self, session):
        """
        Stores the session data and expiry time in the Riak datastore.

        """
        session_bucket = self.client.bucket('sessions')
        session_object = session_bucket.new(session.sid,
                                            (session.expiry,
                                             json.dumps(dict(session),
                                                        cls=Encoder)))
        session_object.store()

    def get(self, cookie, app, ip, user_agent):
        """
        Gets the session data from the Riak datastore and compares the HMAC
        signatures of the IP and User-Agent against the session ID to ensure
        it has not been altered.

        """
        sid = self.generate_sid(app, ip, user_agent)

        if cookie != sid:
            raise SessionValidationError('Tampered session.')

        session_bucket = self.client.bucket('sessions')
        stored_value = session_bucket.get(sid)
        if stored_value:
            expiry, serialized = stored_value.get_data()
            if serialized and (expiry is None or expiry > datetime.now()):
                data = json.loads(serialized, object_hook=date_hook)
                return RiakSession(data, sid=sid, expiry=expiry)

        session = RiakSession(sid=sid)
        session.expiry = self.get_expiration_time(app, session)
        return session

    def open_session(self, app, request):
        """
        Fetch the session from the Riak database after validating the session ID
        or create a new session ID + session.

        """
        cookie = request.cookies.get(app.session_cookie_name)
        ip = request.remote_addr
        # Flask test server has no User-Agent, so return safe value here
        user_agent = request.headers.get('User-Agent', 'NULL')

        if cookie:
            try:
                session = self.get(cookie, app, ip, user_agent)
                return session
            except SessionValidationError:
                # we'll just pass and generate a new session
                LOG.exception('Invalid session')

        sid = self.generate_sid(app, ip, user_agent)
        session = RiakSession(sid=sid, new=True)
        session.expiry = self.get_expiration_time(app, session)
        return session

    def save_session(self, app, session, response):
        """
        This will be called by Flask during request teardown.  Saves the session
        if it has been modified.

        """
        domain = self.get_cookie_domain(app)
        if not session:
            response.delete_cookie(app.session_cookie_name, domain)
            return

        if not session.modified:
            return

        self.put(session)
        cookie_contents = session.sid
        response.set_cookie(app.session_cookie_name, cookie_contents,
                            expires=session.expiry,
                            httponly=self.get_cookie_httponly(app),
                            secure=self.get_cookie_secure(app),
                            domain=domain)

class RiakSessions(object):
    """
    Sets up the Flask-RiakSessions extension during application startup.
    Instantiate this class and pass the application to the `app` argument to
    set it up.

    """
    def __init__(self, app=None):
        self.app = app
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        """
        Sets up default config variables and instantiates the session interface.

        """
        app.config.setdefault('RIAK_BIN_DIR', os.environ.get('RIAK_DIR',''))
        app.config.setdefault('RIAK_HTTP_PORT', 10018)
        app.config.setdefault('RIAK_PROTOBUFS_PORT', 9001)
        app.config.setdefault('RIAK_SESSIONS_HASH_FUNCTION', hashlib.sha256)

        app.session_interface = RiakSessionInterface(app)
<commit_msg>Use unique token as part of SID<commit_after># -*- coding: utf-8 -*-
"""
    flask.ext.riaksessions
    ----------------------

    This module provides a Riak-backed session store for Flask sessions.

    :copyright: (c) 2013 Tim Gross
    :license: MIT

"""

__version_info__ = ('0', '1', '0')
__version__ = '.'.join(__version_info__)
__author__ = 'Tim Gross'
__license__ = 'MIT'
__copyright__ = '(c) 2013 Tim Gross'
__all__ = ['RiakSessions', 'RiakSessionInterface', 'RiakSession']

import base64
from datetime import datetime
import hmac
import hashlib
import json
import logging
import os
from uuid import uuid4

import riak
from werkzeug.datastructures import CallbackDict
from flask.sessions import SessionInterface, SessionMixin

# shut up about short parameter names
# pylint: disable=C0103
LOG = logging.getLogger(__name__)

def get_hmac(app, content):
    """
    Generates the hash message authentication code (HMAC) using the application
    configuration and the passed content. The content should include a salt.
    Takes the RIAK_SESSIONS_HASH_FUNCTION and SECRET_KEY app config.
    """
    hash_function = app.config['RIAK_SESSIONS_HASH_FUNCTION']
    secret = app.config['SECRET_KEY']
    return base64.b64encode(hmac.new(secret, content, hash_function).digest())

class Encoder(json.JSONEncoder):
    """
    Subclasses JSONEncoder to handle datetimes.
    """

    def default(self, obj):
        """
        Encodes datetime objects in annotated ISO-format:
        '/Date(2013-06-25T23:57:32.779968)/'
        """
        if isinstance(obj, datetime):
            return u'/Date({})/'.format(obj.isoformat())
        return super(Encoder, self).default(self, obj)

def date_hook(data):
    """
    Decodes strings that match the pattern created by the Endoder into datetime
    objects.
    """
    for key, val in data.items():
        if val.startswith('/Date('):
            data[key] = datetime.strptime(val.lstrip('/Date(').rstrip(')/'),
                                          "%Y-%m-%dT%H:%M:%S.%f")
    return data

class SessionValidationError(Exception):
    """
    Raised in the event a session id cannot be validated and should ignored.
    """
    pass

class RiakSession(CallbackDict, SessionMixin):
    """
    Implements a Flask session that can be HMAC-signed.
    """

    def __init__(self, initial=None, sid=None, token=None, new=False, expiry=None):
        def on_update(self):
            """
            Set the dirty-session flag.
            """
            self.modified = True

        CallbackDict.__init__(self, initial, on_update)
        self.sid = sid
        self.token = token
        self.new = new
        self.modified = False
        self.expiry = expiry

    @property
    def key(self):
        return '{}!{}'.format(self.token, self.sid)

class RiakSessionInterface(SessionInterface):
    """
    Implements a Flask session interface that uses Riak as the backing store
    and HMAC-signs the session cookie.
    """

    def __init__(self, app, client=None):
        if client is None:
            client = riak.RiakClient(port=app.config['RIAK_HTTP_PORT'])
        self.client = client

    def generate_sid(self, app, ip, user_agent):
        """
        Uses the IP and user-agent as the inputs to an HMAC to be used as the
        unique session ID.

        """
        sid = get_hmac(app, '{}.{}'.format(ip, base64.b64encode(user_agent)))
        return sid

    def put(self, session):
        """
        Stores the session data and expiry time in the Riak datastore.

        """
        session_bucket = self.client.bucket('sessions')
        session_object = session_bucket.new(session.key,
                                            (session.expiry,
                                             json.dumps(dict(session),
                                                        cls=Encoder)))
        session_object.store()

    def get(self, cookie, app, ip, user_agent):
        """
        Gets the session data from the Riak datastore and compares the HMAC
        signatures of the IP and User-Agent against the session ID to ensure
        it has not been altered.

        """
        sid = self.generate_sid(app, ip, user_agent)
        token, cookie_sid = cookie.split('!', 1)

        if cookie_sid != sid:
            raise SessionValidationError('Tampered session.')

        session_bucket = self.client.bucket('sessions')
        stored_value = session_bucket.get(cookie)
        if stored_value:
            expiry, serialized = stored_value.get_data()
            if serialized and (expiry is None or expiry > datetime.now()):
                data = json.loads(serialized, object_hook=date_hook)
                return RiakSession(data, sid=sid, token=token, expiry=expiry)

        session = RiakSession(sid=sid, token=uuid4(), new=True)
        session.expiry = self.get_expiration_time(app, session)
        return session

    def open_session(self, app, request):
        """
        Fetch the session from the Riak database after validating the session ID
        or create a new session ID + session.

        """
        cookie = request.cookies.get(app.session_cookie_name)
        ip = request.remote_addr
        # Flask test server has no User-Agent, so return safe value here
        user_agent = request.headers.get('User-Agent', 'NULL')

        if cookie:
            try:
                session = self.get(cookie, app, ip, user_agent)
                return session
            except SessionValidationError:
                # we'll just pass and generate a new session
                LOG.exception('Invalid session')

        sid = self.generate_sid(app, ip, user_agent)
        session = RiakSession(sid=sid, token=uuid4(), new=True)
        session.expiry = self.get_expiration_time(app, session)
        return session

    def save_session(self, app, session, response):
        """
        This will be called by Flask during request teardown.  Saves the session
        if it has been modified.

        """
        domain = self.get_cookie_domain(app)
        if not session:
            response.delete_cookie(app.session_cookie_name, domain)
            return

        if not session.modified:
            return

        self.put(session)
        cookie_contents = session.key
        response.set_cookie(app.session_cookie_name, cookie_contents,
                            expires=session.expiry,
                            httponly=self.get_cookie_httponly(app),
                            secure=self.get_cookie_secure(app),
                            domain=domain)

class RiakSessions(object):
    """
    Sets up the Flask-RiakSessions extension during application startup.
    Instantiate this class and pass the application to the `app` argument to
    set it up.

    """
    def __init__(self, app=None):
        self.app = app
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        """
        Sets up default config variables and instantiates the session interface.

        """
        app.config.setdefault('RIAK_BIN_DIR', os.environ.get('RIAK_DIR',''))
        app.config.setdefault('RIAK_HTTP_PORT', 10018)
        app.config.setdefault('RIAK_PROTOBUFS_PORT', 9001)
        app.config.setdefault('RIAK_SESSIONS_HASH_FUNCTION', hashlib.sha256)

        app.session_interface = RiakSessionInterface(app)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import re, time, codecs, argparse, sys
import pprint
import CommonMark

class colors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

def trace_calls(frame, event, arg):
	co = frame.f_code
	func_name = co.co_name
	if func_name == "write":
		return
	line_no = frame.f_lineno
	filename = co.co_filename
	if event == "call" and not re.match("__", func_name) and re.search("CommonMark", filename):
		print("Called "+func_name+" at "+str(line_no)+" in "+filename)
		return trace_calls

parser = argparse.ArgumentParser(description="script to run the CommonMark specification tests against the CommonMark.py parser")
parser.add_argument('-t', help="Single test to run or comma seperated list of tests (-t 10 or -t 10,11,12,13)")
parser.add_argument('-v', action="store_true", help="Verbose mode for debugging, print more stuff...")
parser.add_argument('-i', action="store_true", help="Interactive Markdown input mode")
parser.add_argument('-d', action="store_true", help="Debug stuff")
args = parser.parse_args()

if args.d:
	sys.settrace(trace_calls)

renderer = CommonMark.HTMLRenderer()
parser = CommonMark.DocParser()

f = codecs.open("spec.txt", encoding="utf-8")
datalist = []
for line in f:
	datalist.append(line)
data = "\n".join(datalist)

passed = 0
failed = 0
examples = []
example_number = 0
current_section = ""
tabChar = u'\u2192'
spaceChar = u'\u2423'

def showSpaces(s):
	s = s
	#t = str(s[:])
	t = s
	t = re.sub(u"\\t", tabChar, t)
	t = re.sub(u" ", spaceChar, t)
	return t

t = re.sub(r"\r\n?", r"\n", data)
tests = re.sub(r"^<!-- END TESTS -->(.|[\n])*", '', t, flags=re.M)
testMatch = re.findall(re.compile(r"^\.\n([\s\S]*?)^\.\n([\s\S]*?)^\.$|^#{1,6} *(.*)$", re.M), tests)

for match in testMatch:
	if not match[2] == "":
		current_section = match[2]
	else:
		example_number += 1
		examples.append({'markdown': match[0], 'html': match[1], 'section': current_section, 'number': example_number})

current_section = ""

startTime = time.clock()

if args.i:
	while True:
		s = raw_input(colors.OKBLUE+"Markdown: "+colors.ENDC)
		ast = parser.parse(s)
		html = renderer.render(ast)
		print(colors.WARNING+"="*10+"AST====="+colors.ENDC)
		parser.dumpAST(ast)
		print(colors.WARNING+"="*10+"HTML===="+colors.ENDC)
		print(html)
	exit(0)

# some tests?
if args.t:
	tests = args.t.split(",")
	choice_examples = []
	for t in tests:
		if not t == "" and len(examples) > int(t):
			choice_examples.append(examples[int(t)-1])
	examples = choice_examples

# all tests

for i, example in enumerate(examples): # [0,examples[0]]
	if not example['section'] == "" and not current_section == example['section']:
		print(colors.HEADER+"\n"+example['section']+colors.ENDC)
		current_section = example['section']

	print("Test #"+str(i+1))
	ast = parser.parse(re.sub(tabChar, "\t", example['markdown']))
	actual = renderer.render(ast)
	if actual == example['html']:
		passed += 1
		print(colors.OKGREEN+"\ntick"+colors.ENDC)
		if args.v:
			print(colors.OKBLUE+"=== markdown ===============\n"+colors.ENDC+showSpaces(example['markdown'])+colors.OKBLUE+"\n=== expected ===============\n"+colors.ENDC+showSpaces(example['html'])+colors.OKBLUE+"\n=== got ====================\n"+colors.ENDC+showSpaces(actual))
	else:
		failed += 1
		print(colors.FAIL+"\ncross"+colors.ENDC)
		print(colors.WARNING+"=== markdown ===============\n"+colors.ENDC+showSpaces(example['markdown'])+colors.WARNING+"\n=== expected ===============\n"+colors.ENDC+showSpaces(example['html'])+colors.WARNING+"\n=== got ====================\n"+colors.ENDC+showSpaces(actual))

print(str(passed)+" tests passed, "+str(failed)+" failed")

endTime = time.clock()
runTime = endTime-startTime

print("runtime: "+str(runTime)+"s")
<commit_msg>dumpAST again...<commit_after>#!/usr/bin/env python

import re, time, codecs, argparse, sys
import pprint
import CommonMark

class colors:
    HEADER = '\033[95m'
    OKBLUE = '\033[94m'
    OKGREEN = '\033[92m'
    WARNING = '\033[93m'
    FAIL = '\033[91m'
    ENDC = '\033[0m'

def trace_calls(frame, event, arg):
	co = frame.f_code
	func_name = co.co_name
	if func_name == "write":
		return
	line_no = frame.f_lineno
	filename = co.co_filename
	if event == "call" and not re.match("__", func_name) and re.search("CommonMark", filename) and not func_name == "dumpAST":
		print("Called "+func_name+" at "+str(line_no)+" in "+filename)
		return trace_calls

parser = argparse.ArgumentParser(description="script to run the CommonMark specification tests against the CommonMark.py parser")
parser.add_argument('-t', help="Single test to run or comma seperated list of tests (-t 10 or -t 10,11,12,13)")
parser.add_argument('-v', action="store_true", help="Verbose mode for debugging, print more stuff...")
parser.add_argument('-i', action="store_true", help="Interactive Markdown input mode")
parser.add_argument('-d', action="store_true", help="Debug stuff")
args = parser.parse_args()

if args.d:
	sys.settrace(trace_calls)

renderer = CommonMark.HTMLRenderer()
parser = CommonMark.DocParser()

f = codecs.open("spec.txt", encoding="utf-8")
datalist = []
for line in f:
	datalist.append(line)
data = "\n".join(datalist)

passed = 0
failed = 0
examples = []
example_number = 0
current_section = ""
tabChar = u'\u2192'
spaceChar = u'\u2423'

def showSpaces(s):
	s = s
	#t = str(s[:])
	t = s
	t = re.sub(u"\\t", tabChar, t)
	t = re.sub(u" ", spaceChar, t)
	return t

t = re.sub(r"\r\n?", r"\n", data)
tests = re.sub(r"^<!-- END TESTS -->(.|[\n])*", '', t, flags=re.M)
testMatch = re.findall(re.compile(r"^\.\n([\s\S]*?)^\.\n([\s\S]*?)^\.$|^#{1,6} *(.*)$", re.M), tests)

for match in testMatch:
	if not match[2] == "":
		current_section = match[2]
	else:
		example_number += 1
		examples.append({'markdown': match[0], 'html': match[1], 'section': current_section, 'number': example_number})

current_section = ""

startTime = time.clock()

if args.i:
	while True:
		s = raw_input(colors.OKBLUE+"Markdown: "+colors.ENDC)
		ast = parser.parse(s)
		html = renderer.render(ast)
		print(colors.WARNING+"="*10+"AST====="+colors.ENDC)
		parser.dumpAST(ast)
		print(colors.WARNING+"="*10+"HTML===="+colors.ENDC)
		print(html)
	exit(0)

# some tests?
if args.t:
	tests = args.t.split(",")
	choice_examples = []
	for t in tests:
		if not t == "" and len(examples) > int(t):
			choice_examples.append(examples[int(t)-1])
	examples = choice_examples

# all tests

for i, example in enumerate(examples): # [0,examples[0]]
	if not example['section'] == "" and not current_section == example['section']:
		print(colors.HEADER+"\n"+example['section']+colors.ENDC)
		current_section = example['section']

	print("Test #"+str(i+1))
	ast = parser.parse(re.sub(tabChar, "\t", example['markdown']))
	actual = renderer.render(ast)
	if actual == example['html']:
		passed += 1
		print(colors.OKGREEN+"\ntick"+colors.ENDC)
		if args.v:
			print(colors.OKBLUE+"=== markdown ===============\n"+colors.ENDC+showSpaces(example['markdown'])+colors.OKBLUE+"\n=== expected ===============\n"+colors.ENDC+showSpaces(example['html'])+colors.OKBLUE+"\n=== got ====================\n"+colors.ENDC+showSpaces(actual))
	else:
		failed += 1
		print(colors.FAIL+"\ncross"+colors.ENDC)
		print(colors.WARNING+"=== markdown ===============\n"+colors.ENDC+showSpaces(example['markdown'])+colors.WARNING+"\n=== expected ===============\n"+colors.ENDC+showSpaces(example['html'])+colors.WARNING+"\n=== got ====================\n"+colors.ENDC+showSpaces(actual))

print(str(passed)+" tests passed, "+str(failed)+" failed")

endTime = time.clock()
runTime = endTime-startTime

print("runtime: "+str(runTime)+"s")
<|endoftext|>


<commit_before>"""Tests for automatic discovery of modules"""
import os
import sys
import shutil
import tempfile
import platform

import pytest

from setuptools.discovery import ModuleFinder, FlatLayoutModuleFinder

from .test_find_packages import has_symlink, ensure_files

class TestModuleFinder:
    def find(self, path, *args, **kwargs):
        return set(ModuleFinder.find(str(path), *args, **kwargs))

    EXAMPLES = {
        # circumstance: (files, kwargs, expected_modules)
        "simple_folder": (
            ["file.py", "other.py"],
            {},  # kwargs
            ["file", "other"],
        ),
        "exclude": (
            ["file.py", "other.py"],
            {"exclude": ["f*"]},
            ["other"],
        ),
        "include": (
            ["file.py", "fole.py", "other.py"],
            {"include": ["f*"], "exclude": ["fo*"]},
            ["file"],
        ),
        "invalid-name": (
            ["my-file.py", "other.file.py"],
            {},
            []
        )
    }

    @pytest.mark.parametrize("example", EXAMPLES.keys())
    def test_finder(self, tmp_path, example):
        files, kwargs, expected_modules = self.EXAMPLES[example]
        ensure_files(tmp_path, files)
        assert self.find(tmp_path, **kwargs) == set(expected_modules)

    @pytest.mark.skipif(not has_symlink(), reason='Symlink support required')
    def test_symlinked_packages_are_included(self, tmp_path):
        src = "_myfiles/file.py"
        ensure_files(tmp_path, [src])
        os.symlink(tmp_path / src, tmp_path / "link.py")
        assert self.find(tmp_path) == {"link"}

class TestFlatLayoutModuleFinder:
    def find(self, path, *args, **kwargs):
        return set(FlatLayoutModuleFinder.find(str(path)))

    EXAMPLES = {
        # circumstance: (files, expected_modules)
        "hidden-files": (
            [".module.py"],
            []
        ),
        "private-modules": (
            ["_module.py"],
            []
        ),
        "common-names": (
            ["setup.py", "conftest.py", "test.py", "tests.py", "example.py", "mod.py"],
            ["mod"]
        ),
        "tool-specific": (
            ["tasks.py", "fabfile.py", "noxfile.py", "dodo.py", "manage.py", "mod.py"],
            ["mod"]
        )
    }

    @pytest.mark.parametrize("example", EXAMPLES.keys())
    def test_unwanted_files_not_included(self, tmp_path, example):
        files, expected_modules = self.EXAMPLES[example]
        ensure_files(tmp_path, files)
        assert self.find(tmp_path) == set(expected_modules)
<commit_msg>Cleanup test imports<commit_after>"""Tests for automatic discovery of modules"""
import os

import pytest

from setuptools.discovery import FlatLayoutModuleFinder, ModuleFinder

from .test_find_packages import ensure_files, has_symlink

class TestModuleFinder:
    def find(self, path, *args, **kwargs):
        return set(ModuleFinder.find(str(path), *args, **kwargs))

    EXAMPLES = {
        # circumstance: (files, kwargs, expected_modules)
        "simple_folder": (
            ["file.py", "other.py"],
            {},  # kwargs
            ["file", "other"],
        ),
        "exclude": (
            ["file.py", "other.py"],
            {"exclude": ["f*"]},
            ["other"],
        ),
        "include": (
            ["file.py", "fole.py", "other.py"],
            {"include": ["f*"], "exclude": ["fo*"]},
            ["file"],
        ),
        "invalid-name": (
            ["my-file.py", "other.file.py"],
            {},
            []
        )
    }

    @pytest.mark.parametrize("example", EXAMPLES.keys())
    def test_finder(self, tmp_path, example):
        files, kwargs, expected_modules = self.EXAMPLES[example]
        ensure_files(tmp_path, files)
        assert self.find(tmp_path, **kwargs) == set(expected_modules)

    @pytest.mark.skipif(not has_symlink(), reason='Symlink support required')
    def test_symlinked_packages_are_included(self, tmp_path):
        src = "_myfiles/file.py"
        ensure_files(tmp_path, [src])
        os.symlink(tmp_path / src, tmp_path / "link.py")
        assert self.find(tmp_path) == {"link"}

class TestFlatLayoutModuleFinder:
    def find(self, path, *args, **kwargs):
        return set(FlatLayoutModuleFinder.find(str(path)))

    EXAMPLES = {
        # circumstance: (files, expected_modules)
        "hidden-files": (
            [".module.py"],
            []
        ),
        "private-modules": (
            ["_module.py"],
            []
        ),
        "common-names": (
            ["setup.py", "conftest.py", "test.py", "tests.py", "example.py", "mod.py"],
            ["mod"]
        ),
        "tool-specific": (
            ["tasks.py", "fabfile.py", "noxfile.py", "dodo.py", "manage.py", "mod.py"],
            ["mod"]
        )
    }

    @pytest.mark.parametrize("example", EXAMPLES.keys())
    def test_unwanted_files_not_included(self, tmp_path, example):
        files, expected_modules = self.EXAMPLES[example]
        ensure_files(tmp_path, files)
        assert self.find(tmp_path) == set(expected_modules)
<|endoftext|>