<commit_before>import json
import requests
from . import check_input_attribute, standard_error_message
from pymisp import MISPAttribute, MISPEvent, MISPObject

misperrors = {'error': 'Error'}
mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'}
moduleinfo = {'version': '1', 'author': 'Christian Studer',
              'description': 'An expansion module to query variotdbs.pl',
              'module-type': ['expansion', 'hover']}
moduleconfig = ['API_key']
variotdbs_url = 'https://www.variotdbs.pl/api'

class VariotdbsParser:
    def __init__(self, attribute):
        misp_attribute = MISPAttribute()
        misp_attribute.from_dict(**attribute)
        misp_event = MISPEvent()
        misp_event.add_attribute(**misp_attribute)
        self.__misp_attribute = misp_attribute
        self.__misp_event = misp_event
        self.__exploit_mapping = {
            'credits': 'credit',
            'exploit': 'exploit'
        }
        self.__exploit_multiple_mapping = {
            'cve': {
                'feature': 'cve_id',
                'relation': 'cve-id'
            },
            'references': {
                'feature': 'url',
                'relation': 'reference'
            }
        }
        self.__vulnerability_data_mapping = {
            'credits': 'credit',
            'description': 'description',
            'title': 'summary'
        }
        self.__vulnerability_flat_mapping = {
            'cve': 'id', 'id': 'id'
        }

    @property
    def exploit_mapping(self) -> dict:
        return self.__exploit_mapping

    @property
    def exploit_multiple_mapping(self) -> dict:
        return self.__exploit_multiple_mapping

    @property
    def misp_attribute(self) -> MISPAttribute:
        return self.__misp_attribute

    @property
    def misp_event(self) -> MISPEvent:
        return self.__misp_event

    @property
    def vulnerability_data_mapping(self) -> dict:
        return self.__vulnerability_data_mapping

    @property
    def vulnerability_flat_mapping(self) -> dict:
        return self.__vulnerability_flat_mapping

    def get_results(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object') if event.get(key)}
        return {'results': results}

    def parse_exploit_information(self, query_results):
        for exploit in query_results['results']:
            exploit_object = MISPObject('exploit')
            exploit_object.add_attribute('exploitdb-id', exploit['edb_id'])
            for feature, relation in self.exploit_mapping.items():
                if exploit.get(feature):
                    exploit_object.add_attribute(
                        relation,
                        exploit[feature]['data']
                    )
            for feature, relation in self.exploit_multiple_mapping.items():
                if exploit.get(feature):
                    for value in exploit[feature]['data']:
                        exploit_object.add_attribute(
                            relation['relation'],
                            value[relation['feature']]
                        )
            exploit_object.add_reference(self.misp_attribute.uuid, 'related-to')
            self.misp_event.add_object(exploit_object)

    def parse_vulnerability_information(self, query_results):
        vulnerability_object = MISPObject('vulnerability')
        for feature, relation in self.vulnerability_flat_mapping.items():
            if query_results.get(feature):
                vulnerability_object.add_attribute(
                    relation,
                    query_results[feature]
                )
        for feature, relation in self.vulnerability_data_mapping.items():
            if query_results.get(feature, {}).get('data'):
                vulnerability_object.add_attribute(
                    relation,
                    query_results[feature]['data']
                )
        if query_results.get('configurations', {}).get('data'):
            for configuration in query_results['configurations']['data']:
                for node in configuration['nodes']:
                    for cpe_match in node['cpe_match']:
                        if cpe_match['vulnerable']:
                            vulnerability_object.add_attribute(
                                'vulnerable-configuration',
                                cpe_match['cpe23Uri']
                            )
        if query_results.get('cvss', {}).get('data'):
            cvss = {}
            for cvss_data in query_results['cvss']['data']:
                for cvss_v3 in cvss_data['cvssV3']:
                    cvss[float(cvss_v3['trust'])] = cvss_v3
            if cvss:
                cvss = cvss[max(cvss)]
                vulnerability_object.add_attribute(
                    'cvss-score',
                    cvss['baseScore']
                )
                vulnerability_object.add_attribute(
                    'cvss-string',
                    cvss['vectorString']
                )
        if query_results.get('references', {}).get('data'):
            for reference in query_results['references']['data']:
                vulnerability_object.add_attribute(
                    'references',
                    reference['url']
                )
        if query_results.get('sources_release_date', {}).get('data'):
            for release_date in query_results['sources_release_date']['data']:
                if release_date['db'] != 'NVD':
                    continue
                if release_date['id'] == self.misp_attribute.value:
                    vulnerability_object.add_attribute(
                        'published',
                        release_date['date']
                    )
                    break
        if query_results.get('sources_update_date', {}).get('data'):
            for update_date in query_results['sources_update_date']['data']:
                if update_date['db'] != 'NVD':
                    continue
                if update_date['id'] == self.misp_attribute.value:
                    vulnerability_object.add_attribute(
                        'modified',
                        update_date['date']
                    )
                    break
        vulnerability_object.add_reference(self.misp_attribute.uuid, 'related-to')
        self.misp_event.add_object(vulnerability_object)

def handler(q=False):
    if q is False:
        return False
    request = json.loads(q)
    if not request.get('attribute') or not check_input_attribute(request['attribute']):
        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
    attribute = request['attribute']
    if attribute.get('type') != 'vulnerability':
        return {'error': 'Vulnerability id missing.'}
    headers = {'Content-Type': 'application/json'}
    if request.get('config', {}).get('API_key'):
        headers['Authorization'] = f"Token {request['config']['API_key']}"
    empty = True
    parser = VariotdbsParser(attribute)
    r = requests.get(f"{variotdbs_url}/vuln/{attribute['value']}/", headers=headers)
    if r.status_code == 200:
        vulnerability_results = r.json()
        if vulnerability_results:
            parser.parse_vulnerability_information(vulnerability_results)
            empty = False
    else:
        if r.reason != 'Not Found':
            return {'error': 'Error while querying the variotdbs API.'}
    r = requests.get(f"{variotdbs_url}/exploits/?cve={attribute['value']}", headers=headers)
    if r.status_code == 200:
        exploit_results = r.json()
        if exploit_results:
            parser.parse_exploit_information(exploit_results)
            empty = False
    else:
        return {'error': 'Error while querying the variotdbs API.'}
    if empty:
        return {'error': 'Empty results'}
    return parser.get_results()

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo
<commit_msg>fix: [variodbs] Properly handling the exploit results when there is more that 10 results<commit_after>import json
import requests
from . import check_input_attribute, standard_error_message
from pymisp import MISPAttribute, MISPEvent, MISPObject

misperrors = {'error': 'Error'}
mispattributes = {'input': ['vulnerability'], 'format': 'misp_standard'}
moduleinfo = {'version': '1', 'author': 'Christian Studer',
              'description': 'An expansion module to query variotdbs.pl',
              'module-type': ['expansion', 'hover']}
moduleconfig = ['API_key']
variotdbs_url = 'https://www.variotdbs.pl/api'

class VariotdbsParser:
    def __init__(self, attribute):
        misp_attribute = MISPAttribute()
        misp_attribute.from_dict(**attribute)
        misp_event = MISPEvent()
        misp_event.add_attribute(**misp_attribute)
        self.__misp_attribute = misp_attribute
        self.__misp_event = misp_event
        self.__exploit_mapping = {
            'credits': 'credit',
            'exploit': 'exploit'
        }
        self.__exploit_multiple_mapping = {
            'cve': {
                'feature': 'cve_id',
                'relation': 'cve-id'
            },
            'references': {
                'feature': 'url',
                'relation': 'reference'
            }
        }
        self.__vulnerability_data_mapping = {
            'credits': 'credit',
            'description': 'description',
            'title': 'summary'
        }
        self.__vulnerability_flat_mapping = {
            'cve': 'id', 'id': 'id'
        }

    @property
    def exploit_mapping(self) -> dict:
        return self.__exploit_mapping

    @property
    def exploit_multiple_mapping(self) -> dict:
        return self.__exploit_multiple_mapping

    @property
    def misp_attribute(self) -> MISPAttribute:
        return self.__misp_attribute

    @property
    def misp_event(self) -> MISPEvent:
        return self.__misp_event

    @property
    def vulnerability_data_mapping(self) -> dict:
        return self.__vulnerability_data_mapping

    @property
    def vulnerability_flat_mapping(self) -> dict:
        return self.__vulnerability_flat_mapping

    def get_results(self):
        event = json.loads(self.misp_event.to_json())
        results = {key: event[key] for key in ('Attribute', 'Object') if event.get(key)}
        return {'results': results}

    def parse_exploit_information(self, query_results):
        for exploit in query_results:
            exploit_object = MISPObject('exploit')
            exploit_object.add_attribute('exploitdb-id', exploit['edb_id'])
            for feature, relation in self.exploit_mapping.items():
                if exploit.get(feature):
                    exploit_object.add_attribute(
                        relation,
                        exploit[feature]['data']
                    )
            for feature, relation in self.exploit_multiple_mapping.items():
                if exploit.get(feature):
                    for value in exploit[feature]['data']:
                        exploit_object.add_attribute(
                            relation['relation'],
                            value[relation['feature']]
                        )
            exploit_object.add_reference(self.misp_attribute.uuid, 'related-to')
            self.misp_event.add_object(exploit_object)

    def parse_vulnerability_information(self, query_results):
        vulnerability_object = MISPObject('vulnerability')
        for feature, relation in self.vulnerability_flat_mapping.items():
            if query_results.get(feature):
                vulnerability_object.add_attribute(
                    relation,
                    query_results[feature]
                )
        for feature, relation in self.vulnerability_data_mapping.items():
            if query_results.get(feature, {}).get('data'):
                vulnerability_object.add_attribute(
                    relation,
                    query_results[feature]['data']
                )
        if query_results.get('configurations', {}).get('data'):
            for configuration in query_results['configurations']['data']:
                for node in configuration['nodes']:
                    for cpe_match in node['cpe_match']:
                        if cpe_match['vulnerable']:
                            vulnerability_object.add_attribute(
                                'vulnerable-configuration',
                                cpe_match['cpe23Uri']
                            )
        if query_results.get('cvss', {}).get('data'):
            cvss = {}
            for cvss_data in query_results['cvss']['data']:
                for cvss_v3 in cvss_data['cvssV3']:
                    cvss[float(cvss_v3['trust'])] = cvss_v3
            if cvss:
                cvss = cvss[max(cvss)]
                vulnerability_object.add_attribute(
                    'cvss-score',
                    cvss['baseScore']
                )
                vulnerability_object.add_attribute(
                    'cvss-string',
                    cvss['vectorString']
                )
        if query_results.get('references', {}).get('data'):
            for reference in query_results['references']['data']:
                vulnerability_object.add_attribute(
                    'references',
                    reference['url']
                )
        if query_results.get('sources_release_date', {}).get('data'):
            for release_date in query_results['sources_release_date']['data']:
                if release_date['db'] != 'NVD':
                    continue
                if release_date['id'] == self.misp_attribute.value:
                    vulnerability_object.add_attribute(
                        'published',
                        release_date['date']
                    )
                    break
        if query_results.get('sources_update_date', {}).get('data'):
            for update_date in query_results['sources_update_date']['data']:
                if update_date['db'] != 'NVD':
                    continue
                if update_date['id'] == self.misp_attribute.value:
                    vulnerability_object.add_attribute(
                        'modified',
                        update_date['date']
                    )
                    break
        vulnerability_object.add_reference(self.misp_attribute.uuid, 'related-to')
        self.misp_event.add_object(vulnerability_object)

def handler(q=False):
    if q is False:
        return False
    request = json.loads(q)
    if not request.get('attribute') or not check_input_attribute(request['attribute']):
        return {'error': f'{standard_error_message}, which should contain at least a type, a value and an uuid.'}
    attribute = request['attribute']
    if attribute.get('type') != 'vulnerability':
        return {'error': 'Vulnerability id missing.'}
    headers = {'Content-Type': 'application/json'}
    if request.get('config', {}).get('API_key'):
        headers['Authorization'] = f"Token {request['config']['API_key']}"
    empty = True
    parser = VariotdbsParser(attribute)
    r = requests.get(f"{variotdbs_url}/vuln/{attribute['value']}/", headers=headers)
    if r.status_code == 200:
        vulnerability_results = r.json()
        if vulnerability_results:
            parser.parse_vulnerability_information(vulnerability_results)
            empty = False
    else:
        if r.reason != 'Not Found':
            return {'error': 'Error while querying the variotdbs API.'}
    r = requests.get(f"{variotdbs_url}/exploits/?cve={attribute['value']}", headers=headers)
    if r.status_code == 200:
        exploit_results = r.json()
        if exploit_results:
            parser.parse_exploit_information(exploit_results['results'])
            empty = False
        if exploit_results['next'] is not None:
            while(1):
                exploit_results = requests.get(exploit_results['next'], headers=headers)
                if exploit_results.status_code != 200:
                    break
                exploit_results = exploit_results.json()
                parser.parse_exploit_information(exploit_results['results'])
                if exploit_results['next'] is None:
                    break
    else:
        return {'error': 'Error while querying the variotdbs API.'}
    if empty:
        return {'error': 'Empty results'}
    return parser.get_results()

def introspection():
    return mispattributes

def version():
    moduleinfo['config'] = moduleconfig
    return moduleinfo
<|endoftext|>


<commit_before>from django.conf import settings
from mediagenerator.generators.bundles.base import Filter
from subprocess import Popen, PIPE

class YUICompressor(Filter):
    def __init__(self, **kwargs):
        super(YUICompressor, self).__init__(**kwargs)
        assert self.filetype in ('css', 'js'), (
            'YUICompressor only supports compilation to css and js. '
            'The parent filter expects "%s".' % self.filetype)

    def get_output(self, variation):
        for input in self.get_input(variation):
            try:
                compressor = settings.YUICOMPRESSOR_PATH
                cmd = Popen(['java', '-jar', compressor,
                             '--charset', 'utf-8', '--type', self.filetype],
                            stdin=PIPE, stdout=PIPE, stderr=PIPE,
                            universal_newlines=True)
                output, error = cmd.communicate(input)
                assert cmd.wait() == 0, 'Command returned bad result:\n%s' % error
                yield output
            except Exception, e:
                raise ValueError("Failed to execute Java VM or yuicompressor. "
                    "Please make sure that you have installed Java "
                    "and that it's in your PATH and that you've configured "
                    "YUICOMPRESSOR_PATH in your settings correctly.\n"
                    "Error was: %s" % e)
<commit_msg>moved subprocess import in yuicompressor to get_output, so pure App Engine users (without Django-nonrel) won't run into problems<commit_after>from django.conf import settings
from mediagenerator.generators.bundles.base import Filter

class YUICompressor(Filter):
    def __init__(self, **kwargs):
        super(YUICompressor, self).__init__(**kwargs)
        assert self.filetype in ('css', 'js'), (
            'YUICompressor only supports compilation to css and js. '
            'The parent filter expects "%s".' % self.filetype)

    def get_output(self, variation):
        # We import this here, so App Engine Helper users don't get import
        # errors.
        from subprocess import Popen, PIPE
        for input in self.get_input(variation):
            try:
                compressor = settings.YUICOMPRESSOR_PATH
                cmd = Popen(['java', '-jar', compressor,
                             '--charset', 'utf-8', '--type', self.filetype],
                            stdin=PIPE, stdout=PIPE, stderr=PIPE,
                            universal_newlines=True)
                output, error = cmd.communicate(input)
                assert cmd.wait() == 0, 'Command returned bad result:\n%s' % error
                yield output
            except Exception, e:
                raise ValueError("Failed to execute Java VM or yuicompressor. "
                    "Please make sure that you have installed Java "
                    "and that it's in your PATH and that you've configured "
                    "YUICOMPRESSOR_PATH in your settings correctly.\n"
                    "Error was: %s" % e)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
Module containing the available commands of the game, the factory class to
create the commands...
Today, the available commands are:
- look,
- talk,
- move,
- enter,
- exit,
- take,
- drop,
- inventory,
- stats,
- help,
- quit,
- attack
- save
"""
import core.command
from core.commands import look, talk, move, enter, exit, take, drop, inventory,\
	help, open, stats, attack, save
from core.localisation import _
import sys

"""
Code corresponding to the quit command
"""
quit = -1

class factory:
	"""
	Class to instanciate a command from a string.
	"""

	"""
	Available commands stored in a dict with as key, the commands and as value,
	the command class to execute.
	"""
	mapping = {
		_('LOOK_COMMAND'): {'allowed_while_fighting': True, 'command': 'look'},
		_('TALK_COMMAND'): {'allowed_while_fighting': False, 'command': 'talk'},
		_('MOVE_COMMAND'): {'allowed_while_fighting': True, 'command': 'move'},
		_('ENTER_COMMAND'): {'allowed_while_fighting': True, 'command': 'enter'},
		_('EXIT_COMMAND'): {'allowed_while_fighting': True, 'command': 'exit'},
		_('TAKE_COMMAND'): {'allowed_while_fighting': False, 'command': 'take'},
		_('DROP_COMMAND'): {'allowed_while_fighting': False, 'command': 'drop'},
		_('INVENTORY_COMMAND'): {'allowed_while_fighting': True, 'command': 'inventory'},
		_('INVENTORY_SHORT_COMMAND'): {'allowed_while_fighting': True, 'command': 'inventory'},
		_('STATS_COMMAND'): {'allowed_while_fighting': True, 'command': 'stats'},
		_('HELP_COMMAND'): {'allowed_while_fighting': True, 'command': 'help'},
		_('OPEN_COMMAND'): {'allowed_while_fighting': False, 'command': 'open'},
		_('SAVE_COMMAND'): {'allowed_while_fighting': False, 'command': 'save'},
		_('ATTACK_COMMAND'): {'allowed_while_fighting': True, 'command': 'attack'}
	}

	mapping_anonymous = {
		'create-player': 'createPlayer'
	}

	@staticmethod
	def create(p, commandFull, savedGameId=None):
		"""
		command.factory.create(p, commandFull, savedGameId=None) -> command.command

		Create the desired command.

		@param p player.player Current player.
		@param commandFull list command to run, the first element of the list
			is the command, the other elements are the command's arguments.

		@return the created command
		"""

		cmd = commandFull[0]
		del commandFull[0]

		if cmd in (_('QUIT_COMMAND'), _('QUIT_SHORT_COMMAND')):
			return quit
		elif p.isConnected() and cmd in factory.mapping.keys():
			cmd = factory.mapping[cmd]
			module = sys.modules['core.commands.' + cmd['command']]

			if p.isFighting() and not cmd['allowed_while_fighting']:
				raise core.command.exception(_('ERROR_DENIED_COMMAND_WHILE_FIGHTING'))

			cmd = getattr(module, cmd['command'])()
		elif cmd in factory.mapping_anonymous.keys():
			module = sys.modules['core.commands.' + factory.mapping_anonymous[cmd]]
			cmd = getattr(module, factory.mapping_anonymous[cmd])()
		else:
			raise core.command.exception(_('ERROR_UNKNOWN_COMMAND'))

		cmd.setArgs(commandFull)
		cmd.setPlayer(p)
		cmd.setSavedGameId(savedGameId)
		return cmd
<commit_msg>anonymous commands removed<commit_after># -*- coding: utf-8 -*-

"""
Module containing the available commands of the game, the factory class to
create the commands...
Today, the available commands are:
- look,
- talk,
- move,
- enter,
- exit,
- take,
- drop,
- inventory,
- stats,
- help,
- quit,
- attack
- save
"""
import core.command
from core.commands import look, talk, move, enter, exit, take, drop, inventory,\
	help, open, stats, attack, save
from core.localisation import _
import sys

"""
Code corresponding to the quit command
"""
quit = -1

class factory:
	"""
	Class to instanciate a command from a string.
	"""

	"""
	Available commands stored in a dict with as key, the commands and as value,
	the command class to execute.
	"""
	mapping = {
		_('LOOK_COMMAND'): {'allowed_while_fighting': True, 'command': 'look'},
		_('TALK_COMMAND'): {'allowed_while_fighting': False, 'command': 'talk'},
		_('MOVE_COMMAND'): {'allowed_while_fighting': True, 'command': 'move'},
		_('ENTER_COMMAND'): {'allowed_while_fighting': True, 'command': 'enter'},
		_('EXIT_COMMAND'): {'allowed_while_fighting': True, 'command': 'exit'},
		_('TAKE_COMMAND'): {'allowed_while_fighting': False, 'command': 'take'},
		_('DROP_COMMAND'): {'allowed_while_fighting': False, 'command': 'drop'},
		_('INVENTORY_COMMAND'): {'allowed_while_fighting': True, 'command': 'inventory'},
		_('INVENTORY_SHORT_COMMAND'): {'allowed_while_fighting': True, 'command': 'inventory'},
		_('STATS_COMMAND'): {'allowed_while_fighting': True, 'command': 'stats'},
		_('HELP_COMMAND'): {'allowed_while_fighting': True, 'command': 'help'},
		_('OPEN_COMMAND'): {'allowed_while_fighting': False, 'command': 'open'},
		_('SAVE_COMMAND'): {'allowed_while_fighting': False, 'command': 'save'},
		_('ATTACK_COMMAND'): {'allowed_while_fighting': True, 'command': 'attack'}
	}

	@staticmethod
	def create(p, commandFull, savedGameId=None):
		"""
		command.factory.create(p, commandFull, savedGameId=None) -> command.command

		Create the desired command.

		@param p player.player Current player.
		@param commandFull list command to run, the first element of the list
			is the command, the other elements are the command's arguments.

		@return the created command
		"""

		cmd = commandFull[0]
		del commandFull[0]

		if cmd in (_('QUIT_COMMAND'), _('QUIT_SHORT_COMMAND')):
			return quit
		elif p.isConnected() and cmd in factory.mapping.keys():
			cmd = factory.mapping[cmd]
			module = sys.modules['core.commands.' + cmd['command']]

			if p.isFighting() and not cmd['allowed_while_fighting']:
				raise core.command.exception(_('ERROR_DENIED_COMMAND_WHILE_FIGHTING'))

			cmd = getattr(module, cmd['command'])()
		else:
			raise core.command.exception(_('ERROR_UNKNOWN_COMMAND'))

		cmd.setArgs(commandFull)
		cmd.setPlayer(p)
		cmd.setSavedGameId(savedGameId)
		return cmd
<|endoftext|>


<commit_before>import os

import sublime
from sublime_plugin import WindowCommand, TextCommand

from ..git_command import GitCommand

COMMIT_HELP_TEXT = """

## To make a commit, type your commit message and press {key}-ENTER. To cancel
## the commit, close the window.

## You may also reference or close a GitHub issue with this commit.  To do so,
## type `#` followed by the `tab` key.  You will be shown a list of issues
## related to the current repo.  You may also type `owner/repo#` plus the `tab`
## key to reference an issue in a different GitHub repo.
""".format(key="CTRL" if os.name == "nt" else "SUPER")

COMMIT_SIGN_TEXT = """

Signed-off-by: {name} <{email}>
"""

COMMIT_TITLE = "COMMIT"

class GsCommitCommand(WindowCommand, GitCommand):

    """
    Display a transient window to capture the user's desired commit message.
    If the user is amending the previous commit, pre-populate the commit
    message area with the previous commit message.
    """

    def run(self, **kwargs):
        sublime.set_timeout_async(lambda: self.run_async(**kwargs), 0)

    def run_async(self, repo_path=None, include_unstaged=False, amend=False):
        repo_path = repo_path or self.repo_path
        view = self.window.new_file()
        view.settings().set("git_savvy.get_long_text_view", True)
        view.settings().set("git_savvy.commit_view.include_unstaged", include_unstaged)
        view.settings().set("git_savvy.commit_view.amend", amend)
        view.settings().set("git_savvy.repo_path", repo_path)
        view.set_syntax_file("Packages/GitSavvy/syntax/make_commit.tmLanguage")
        view.set_name(COMMIT_TITLE)
        view.set_scratch(True)
        view.run_command("gs_commit_initialize_view")

class GsCommitInitializeViewCommand(TextCommand, GitCommand):

    """
    Fill the view with the commit view help message, and optionally
    the previous commit message if amending.
    """

    def run(self, edit):
        merge_msg_path = os.path.join(self.repo_path, ".git", "MERGE_MSG")
        if self.view.settings().get("git_savvy.commit_view.amend"):
            last_commit_message = self.git("log", "-1", "--pretty=%B")
            initial_text = last_commit_message + COMMIT_HELP_TEXT
        elif os.path.exists(merge_msg_path):
            with open(merge_msg_path, "r") as f:
                initial_text = f.read() + COMMIT_HELP_TEXT
        else:
            initial_text = COMMIT_HELP_TEXT

        self.view.run_command("gs_replace_view_text", {
            "text": initial_text,
            "nuke_cursors": True
            })

class GsCommitViewDoCommitCommand(TextCommand, GitCommand):

    """
    Take the text of the current view (minus the help message text) and
    make a commit using the text for the commit message.
    """

    def run(self, edit):
        sublime.set_timeout_async(self.run_async, 0)

    def run_async(self):
        view_text = self.view.substr(sublime.Region(0, self.view.size()))
        commit_message = view_text.replace(COMMIT_HELP_TEXT, "")

        if self.view.settings().get("git_savvy.commit_view.include_unstaged"):
            self.add_all_tracked_files()

        if self.view.settings().get("git_savvy.commit_view.amend"):
            self.git("commit", "-q", "--amend", "-F", "-", stdin=commit_message)
        else:
            self.git("commit", "-q", "-F", "-", stdin=commit_message)

        self.view.window().focus_view(self.view)
        self.view.window().run_command("close_file")

class GsCommitViewSignCommand(TextCommand, GitCommand):

    """
    Sign off the commit with full name and email
    """

    def run(self, edit):
        view_text = self.view.substr(sublime.Region(0, self.view.size()))
        view_text_list = view_text.split(COMMIT_HELP_TEXT)

        config_name = self.git("config", "user.name").strip()
        config_email = self.git("config", "user.email").strip()

        sign_text = COMMIT_SIGN_TEXT.format(name=config_name, email=config_email)
        view_text_list[0] += sign_text
        view_text_list.insert(1, COMMIT_HELP_TEXT)

        self.view.run_command("gs_replace_view_text", {
            "text": "".join(view_text_list),
            "nuke_cursors": True
            })
<commit_msg>Join the list using the help text as a delimiter<commit_after>import os

import sublime
from sublime_plugin import WindowCommand, TextCommand

from ..git_command import GitCommand

COMMIT_HELP_TEXT = """

## To make a commit, type your commit message and press {key}-ENTER. To cancel
## the commit, close the window.

## You may also reference or close a GitHub issue with this commit.  To do so,
## type `#` followed by the `tab` key.  You will be shown a list of issues
## related to the current repo.  You may also type `owner/repo#` plus the `tab`
## key to reference an issue in a different GitHub repo.
""".format(key="CTRL" if os.name == "nt" else "SUPER")

COMMIT_SIGN_TEXT = """

Signed-off-by: {name} <{email}>
"""

COMMIT_TITLE = "COMMIT"

class GsCommitCommand(WindowCommand, GitCommand):

    """
    Display a transient window to capture the user's desired commit message.
    If the user is amending the previous commit, pre-populate the commit
    message area with the previous commit message.
    """

    def run(self, **kwargs):
        sublime.set_timeout_async(lambda: self.run_async(**kwargs), 0)

    def run_async(self, repo_path=None, include_unstaged=False, amend=False):
        repo_path = repo_path or self.repo_path
        view = self.window.new_file()
        view.settings().set("git_savvy.get_long_text_view", True)
        view.settings().set("git_savvy.commit_view.include_unstaged", include_unstaged)
        view.settings().set("git_savvy.commit_view.amend", amend)
        view.settings().set("git_savvy.repo_path", repo_path)
        view.set_syntax_file("Packages/GitSavvy/syntax/make_commit.tmLanguage")
        view.set_name(COMMIT_TITLE)
        view.set_scratch(True)
        view.run_command("gs_commit_initialize_view")

class GsCommitInitializeViewCommand(TextCommand, GitCommand):

    """
    Fill the view with the commit view help message, and optionally
    the previous commit message if amending.
    """

    def run(self, edit):
        merge_msg_path = os.path.join(self.repo_path, ".git", "MERGE_MSG")
        if self.view.settings().get("git_savvy.commit_view.amend"):
            last_commit_message = self.git("log", "-1", "--pretty=%B")
            initial_text = last_commit_message + COMMIT_HELP_TEXT
        elif os.path.exists(merge_msg_path):
            with open(merge_msg_path, "r") as f:
                initial_text = f.read() + COMMIT_HELP_TEXT
        else:
            initial_text = COMMIT_HELP_TEXT

        self.view.run_command("gs_replace_view_text", {
            "text": initial_text,
            "nuke_cursors": True
            })

class GsCommitViewDoCommitCommand(TextCommand, GitCommand):

    """
    Take the text of the current view (minus the help message text) and
    make a commit using the text for the commit message.
    """

    def run(self, edit):
        sublime.set_timeout_async(self.run_async, 0)

    def run_async(self):
        view_text = self.view.substr(sublime.Region(0, self.view.size()))
        commit_message = view_text.replace(COMMIT_HELP_TEXT, "")

        if self.view.settings().get("git_savvy.commit_view.include_unstaged"):
            self.add_all_tracked_files()

        if self.view.settings().get("git_savvy.commit_view.amend"):
            self.git("commit", "-q", "--amend", "-F", "-", stdin=commit_message)
        else:
            self.git("commit", "-q", "-F", "-", stdin=commit_message)

        self.view.window().focus_view(self.view)
        self.view.window().run_command("close_file")

class GsCommitViewSignCommand(TextCommand, GitCommand):

    """
    Sign off the commit with full name and email
    """

    def run(self, edit):
        view_text = self.view.substr(sublime.Region(0, self.view.size()))
        view_text_list = view_text.split(COMMIT_HELP_TEXT)

        config_name = self.git("config", "user.name").strip()
        config_email = self.git("config", "user.email").strip()

        sign_text = COMMIT_SIGN_TEXT.format(name=config_name, email=config_email)
        view_text_list[0] += sign_text

        self.view.run_command("gs_replace_view_text", {
            "text": COMMIT_HELP_TEXT.join(view_text_list),
            "nuke_cursors": True
            })
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import argparse
import subprocess
import tempfile
import os.path
import glob

def main():
    parser = argparse.ArgumentParser(
        description=
        "Tool for comparing database JSON outputs from two database's.")
    parser.add_argument('a_db')
    parser.add_argument('b_db')

    args = parser.parse_args()

    assert os.path.isdir(args.a_db)
    assert os.path.isdir(args.b_db)

    for a_json_in in glob.glob('{}/*.json'.format(args.a_db)):
        a_json_base = os.path.basename(a_json_in)
        b_json_in = '{}/{}'.format(args.b_db, a_json_base)

        if not os.path.exists(b_json_in):
            print('{} not found!'.format(b_json_in))
            continue

        with tempfile.NamedTemporaryFile(suffix="_a_{}".format(
                a_json_base)) as a_json_out, tempfile.NamedTemporaryFile(
                    suffix="_b_{}".format(a_json_base)) as b_json_out:
            subprocess.check_call(
                "python3 -m utils.xjson {}".format(a_json_in), shell=True, stdout=a_json_out)
            subprocess.check_call(
                "python3 -m utils.xjson {}".format(b_json_in), shell=True, stdout=b_json_out)

            print("Comparing {}".format(a_json_base))
            subprocess.call(
                "diff -U 3 {} {}".format(a_json_out.name, b_json_out.name),
                shell=True)

if __name__ == "__main__":
    main()
<commit_msg>Run make format.<commit_after>#!/usr/bin/env python3
import argparse
import subprocess
import tempfile
import os.path
import glob

def main():
    parser = argparse.ArgumentParser(
        description=
        "Tool for comparing database JSON outputs from two database's.")
    parser.add_argument('a_db')
    parser.add_argument('b_db')

    args = parser.parse_args()

    assert os.path.isdir(args.a_db)
    assert os.path.isdir(args.b_db)

    for a_json_in in glob.glob('{}/*.json'.format(args.a_db)):
        a_json_base = os.path.basename(a_json_in)
        b_json_in = '{}/{}'.format(args.b_db, a_json_base)

        if not os.path.exists(b_json_in):
            print('{} not found!'.format(b_json_in))
            continue

        with tempfile.NamedTemporaryFile(suffix="_a_{}".format(
                a_json_base)) as a_json_out, tempfile.NamedTemporaryFile(
                    suffix="_b_{}".format(a_json_base)) as b_json_out:
            subprocess.check_call(
                "python3 -m utils.xjson {}".format(a_json_in),
                shell=True,
                stdout=a_json_out)
            subprocess.check_call(
                "python3 -m utils.xjson {}".format(b_json_in),
                shell=True,
                stdout=b_json_out)

            print("Comparing {}".format(a_json_base))
            subprocess.call(
                "diff -U 3 {} {}".format(a_json_out.name, b_json_out.name),
                shell=True)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
    Python script to upload and replace broken links with new ones.

    Example Usage: python utils/find_replace.py --csv input.csv --out output.csv

    Outputs a csv with meeting_id previous_url previous_filesystem_path new_file_url file_id event_file_id columns
"""
import argparse
import csv
import logging
import os
import shutil
import tempfile
from collections import namedtuple

from pmg import app, db
from pmg.models.resources import CommitteeMeeting, EventFile, File

def create_logger(args):
    logging.basicConfig(level=logging.INFO, filemode="a", encoding="utf-8")
    formatter = logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
    logger = logging.getLogger(__name__)
    file_handler = logging.FileHandler(args.logfile)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    return logger

def read_csv(path):
    with open(path, "r") as f:
        reader = csv.DictReader(f)
        for row in reader:
            if row["possible_duplicate"] == "True":
                continue
            yield row

def find_meeting_by_id(id):
    meeting = db.session.query(CommitteeMeeting).filter_by(id=int(id)).first()
    return meeting

def upload_file(path):
    with tempfile.TemporaryDirectory() as temp_dir:
        temp_path = os.path.join(temp_dir, os.path.basename(path))
        shutil.copyfile(path, temp_path)
        file = File()
        file.from_file_blob(path, temp_path)
        return file

def process_and_upload_files(args):
    logger = create_logger(args)
    csv_row = namedtuple(
        "Row",
        "meeting_id previous_url previous_filesystem_path new_file_url file_id event_file_id",
    )
    for row in read_csv(args.csv):
        row_url = row["url"]
        meeting_id = row["meeting_id"]
        filesystem_path = row["filesystem_path"]
        meeting = find_meeting_by_id(meeting_id)
        if not meeting:
            logger.error("Could not find meeting {}".format(meeting_id))
            continue
        new_file_url = ""
        try:
            with app.app_context():
                file = upload_file(filesystem_path)
                new_file_url = file.url
        except FileNotFoundError:
            logger.error(
                f"Could not find file {filesystem_path} for meeting id: {meeting_id}, {new_file_url}"
            )
            continue
        if new_file_url and meeting.body and row_url in meeting.body:
            meeting.body = meeting.body.replace(row_url, new_file_url)
            event_file = EventFile(event=meeting, file=file)
            db.session.add(file)
            db.session.add(event_file)
            db.session.add(meeting)
            db.session.commit()
            yield csv_row(
                meeting_id,
                row_url,
                row["filesystem_path"],
                new_file_url,
                file.id,
                event_file.id,
            )

def write_to_csv(csv_path, rows):
    with open(csv_path, "a") as f:
        writer = csv.writer(f)
        writer.writerow(
            [
                "meeting_id",
                "previous_url",
                "previous_filesystem_path",
                "new_file_url",
                "file_id",
                "event_file_id",
            ]
        )
        for row in rows:
            writer.writerow(row)

def main(args):
    logger = create_logger(args)
    logger.info("Started")
    write_to_csv(args.out, process_and_upload_files(args))
    logger.info("Finished")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__name__)
    parser.add_argument(
        "--csv", required=True, help="path to csv file with extracted urls"
    )
    parser.add_argument(
        "--out", required=True, help="path to csv file with the results"
    )
    parser.add_argument("--logfile", required=True, help="path to logfile")
    args = parser.parse_args()
    main(args)
<commit_msg>move context to main function<commit_after>"""
    Python script to upload and replace broken links with new ones.

    Example Usage: python utils/find_replace.py --csv data/input.csv --out data/output.csv --logfile data/output.log

    Outputs a csv with meeting_id previous_url previous_filesystem_path new_file_url file_id event_file_id columns
"""
import argparse
import csv
import logging
import os
import shutil
import tempfile
from collections import namedtuple

from pmg import app, db
from pmg.models.resources import CommitteeMeeting, EventFile, File

def create_logger(args):
    logging.basicConfig(level=logging.INFO, filemode="a", encoding="utf-8")
    formatter = logging.Formatter("%(asctime)s | %(levelname)s | %(message)s")
    logger = logging.getLogger(__name__)
    file_handler = logging.FileHandler(args.logfile)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    return logger

def read_csv(path):
    with open(path, "r") as f:
        reader = csv.DictReader(f)
        for row in reader:
            if row["possible_duplicate"] == "True":
                continue
            yield row

def find_meeting_by_id(id):
    meeting = db.session.query(CommitteeMeeting).filter_by(id=int(id)).first()
    return meeting

def upload_file(path):
    with tempfile.TemporaryDirectory() as temp_dir:
        temp_path = os.path.join(temp_dir, os.path.basename(path))
        shutil.copyfile(path, temp_path)
        file = File()
        file.from_file_blob(path, temp_path)
        return file

def process_and_upload_files(args):
    logger = create_logger(args)
    csv_row = namedtuple(
        "Row",
        "meeting_id previous_url previous_filesystem_path new_file_url file_id event_file_id",
    )
    for row in read_csv(args.csv):
        row_url = row["url"]
        meeting_id = row["meeting_id"]
        filesystem_path = row["filesystem_path"]
        meeting = find_meeting_by_id(meeting_id)
        if not meeting:
            logger.error("Could not find meeting {}".format(meeting_id))
            continue
        new_file_url = None
        try:
            file = upload_file(filesystem_path)
            new_file_url = file.url
        except FileNotFoundError:
            logger.error(
                f"Could not find file {filesystem_path} for meeting id: {meeting_id}, {new_file_url}"
            )
            continue
        if new_file_url and meeting.body and row_url in meeting.body:
            meeting.body = meeting.body.replace(row_url, new_file_url)
            event_file = EventFile(event=meeting, file=file)
            db.session.add(file)
            db.session.add(event_file)
            db.session.add(meeting)
            db.session.commit()
            yield csv_row(
                meeting_id,
                row_url,
                row["filesystem_path"],
                new_file_url,
                file.id,
                event_file.id,
            )

def write_to_csv(csv_path, rows):
    with open(csv_path, "a") as f:
        writer = csv.writer(f)
        writer.writerow(
            [
                "meeting_id",
                "previous_url",
                "previous_filesystem_path",
                "new_file_url",
                "file_id",
                "event_file_id",
            ]
        )
        for row in rows:
            writer.writerow(row)

def main(args):
    logger = create_logger(args)
    logger.info("Started")
    write_to_csv(args.out, process_and_upload_files(args))
    logger.info("Finished")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description=__name__)
    parser.add_argument(
        "--csv", required=True, help="path to csv file with extracted urls"
    )
    parser.add_argument(
        "--out", required=True, help="path to csv file with the results"
    )
    parser.add_argument("--logfile", required=True, help="path to logfile")
    args = parser.parse_args()

    with app.app_context():
        main(args)
<|endoftext|>


<commit_before># ===============================================================================
# Copyright 2016 gabe-parrish
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# ============= standard library imports ========================
import os

import datetime
# from datetime import datetime
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd

# ============= local library imports ===========================

def grab(df_subset):
    ''' makes life easy by turning df subset values into arrays.'''

    array = np.array(df_subset.values.tolist())

    return array

def get_columns(dataframe):
    """gets the columns of the whole dataframe. Return a dict of colums with names."""
    tracker_dict = {}

    # for every column index in dataframe, get column and grab(column),\
    #  then store resultant array in a dict['column index'] = array
    cols = list(dataframe)
    for item in cols:
        #print 'column value \n {}'.format(dataframe[item])
        col = grab(dataframe[item])
        tracker_dict['{}'.format(item)] = col
    #print 'tracker dict', tracker_dict

    return tracker_dict

def reader_function(path):
    """
    takes a path, reads that in as a dataframe, then puts it back into a dict.
    """
    read_file = pd.read_csv(path)
    return read_file

def grapher(col_dict, x, y, plot_output, function_count, together=False):
    """
    input: x is a list of independent variables or a time series. y is a list of dependent variables.
    """

    color_list = ['green', 'blue', 'red', 'purple', 'black', 'orange', 'yellow', 'maroon']
    if together == False:

        # We plot everything separately
        for x_item in x:
            count = 0
            print 'col_dict x item', col_dict[x_item]
            if x_item == 'Date':
                date_list = col_dict[x_item].tolist()
                #print 'to stirng', col_dict[x_item].tostring()
                date_list = [datetime.datetime.strptime(item, "%Y-%m-%d") for item in date_list]
                ex = date_list
                #print 'ex', ex
            else:
                ex = col_dict[x_item]
            for y_item in y:
                fig = plt.figure()
                aa = fig.add_subplot(111)
                aa.set_title('tracker uni-plot', fontweight='bold')
                aa.set_xlabel('{}'.format(x_item), style='italic')
                aa.set_ylabel('{}'.format(y_item), style='italic')
                aa.plot(ex, col_dict[y_item], color = color_list[count], label=y_item )
                aa.legend(loc='upper right', frameon=True, prop={'size': 10})
                plt.tight_layout()
                plt.savefig('{}/{}_separate{}{}.png'.format(plot_output, y_item, count, function_count))
                count += 1
    if together == True:
        # We plot everything together.
        for x_item in x:
            count = 0
            fig = plt.figure()
            aa = fig.add_subplot(111)
            aa.set_title('tracker multi-plot', fontweight='bold')
            aa.set_xlabel('{}'.format(x_item), style='italic')
            if x_item == 'Date':
                date_list = col_dict[x_item].tolist()
                #print 'to stirng', col_dict[x_item].tostring()
                date_list = [datetime.datetime.strptime(item, "%Y-%m-%d") for item in date_list]
                ex = date_list
                #print 'ex', ex
            else:
                ex = col_dict[x_item]
            for y_item in y:
                aa.set_ylabel('{}'.format(y_item), style='italic')
                aa.plot(ex, col_dict[y_item], color=color_list[count], label=y_item)
                count += 1
            aa.legend(loc='upper right', frameon=True, prop={'size': 10})
            plt.tight_layout()
            plt.savefig('{}/{}_together{}.png'.format(plot_output, x_item, function_count))

def run_tracker_plot(path, x, y, plot_output, function_count, multi=True):
    """Runs the plotter
    path: complete filepath to a csv
    x: list of strings of the column headings you want as independent variables
    e.g. x = ['Date', 'rzsm']
    y: list of strings of the column headings you want as dependent variables
    e.g. y = ['rain', 'soil_ksat', 'eta', 'rzsm']
    multi = True is default which means the graph makes one graph for each ind var with all dep variables on the same plot
    multi = False means graph will make one ind/dep variable plot for each ind variable and each dep variable.
    eg. three ind vars, three dep vars, 3x3 = 9 graphs total.

    This function makes plots for one csv at a time.
    """

    # now you need to read all the files and return a dict of pandas dataframes.
    dataframe = reader_function(path)
    print 'dataframe', dataframe
    # send in the dict containing the df to a grapher function and get to graphing!
    col_dict = get_columns(dataframe)
    grapher(col_dict, x, y, plot_output, function_count, together=multi)

    print 'complete?!?!'

if __name__ == "__main__":
    run_tracker_plot()

    # TODO find a way to pre-configure what you want to plot, then give x axis stuff (time series) and y axis stuff to grapher.
    # TODO - I must eventually devise a way to use this with the one to one RZSM plots, or make this into an object that can handle something like that.<commit_msg>Lots of new scripts in utilities to do the ETrF adjustment.<commit_after># ===============================================================================
# Copyright 2016 gabe-parrish
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# ============= standard library imports ========================
import os

import datetime
# from datetime import datetime
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd

# ============= local library imports ===========================

def grab(df_subset):
    ''' makes life easy by turning df subset values into arrays.'''

    array = np.array(df_subset.values.tolist())

    return array

def get_columns(dataframe):
    """gets the columns of the whole dataframe. Return a dict of colums with names."""
    tracker_dict = {}

    # for every column index in dataframe, get column and grab(column),\
    #  then store resultant array in a dict['column index'] = array
    cols = list(dataframe)
    for item in cols:
        #print 'column value \n {}'.format(dataframe[item])
        col = grab(dataframe[item])
        tracker_dict['{}'.format(item)] = col
    #print 'tracker dict', tracker_dict

    return tracker_dict

def reader_function(path):
    """
    takes a path, reads that in as a dataframe, then puts it back into a dict.
    """
    read_file = pd.read_csv(path)
    return read_file

def grapher(col_dict, x, y, plot_output, function_count, together=False):
    """
    input: x is a list of independent variables or a time series. y is a list of dependent variables.
    """

    color_list = ['green', 'blue', 'red', 'purple', 'black', 'orange', 'yellow', 'maroon']
    if together == False:

        # We plot everything separately
        for x_item in x:
            count = 0
            print 'col_dict x item', col_dict[x_item]
            if x_item == 'Date':
                date_list = col_dict[x_item].tolist()
                #print 'to stirng', col_dict[x_item].tostring()
                date_list = [datetime.datetime.strptime(item, "%Y-%m-%d") for item in date_list]
                ex = date_list
                #print 'ex', ex
            else:
                ex = col_dict[x_item]
            for y_item in y:
                fig = plt.figure()
                aa = fig.add_subplot(111)
                aa.set_title('tracker uni-plot', fontweight='bold')
                aa.set_xlabel('{}'.format(x_item), style='italic')
                aa.set_ylabel('{}'.format(y_item), style='italic')
                aa.plot(ex, col_dict[y_item], color = color_list[count], label=y_item )
                aa.legend(loc='upper right', frameon=True, prop={'size': 10})
                plt.tight_layout()
                plt.savefig('{}/{}_separate{}{}.png'.format(plot_output, y_item, count, function_count))
                count += 1
    if together == True:
        # We plot everything together. # yes
        for x_item in x:
            count = 0
            fig = plt.figure()
            aa = fig.add_subplot(111)
            aa.set_title('tracker multi-plot', fontweight='bold')
            aa.set_xlabel('{}'.format(x_item), style='italic')
            if x_item == 'Date':
                date_list = col_dict[x_item].tolist()
                #print 'to stirng', col_dict[x_item].tostring()
                date_list = [datetime.datetime.strptime(item, "%Y-%m-%d") for item in date_list]
                ex = date_list
                #print 'ex', ex
            else:
                ex = col_dict[x_item]
            for y_item in y:
                aa.set_ylabel('{}'.format(y_item), style='italic')
                aa.plot(ex, col_dict[y_item], color=color_list[count], label=y_item)
                count += 1
            aa.legend(loc='upper right', frameon=True, prop={'size': 10})
            plt.tight_layout()
            plt.savefig('{}/{}_together{}.png'.format(plot_output, x_item, function_count))

def run_tracker_plot(path, x, y, plot_output, function_count, multi=True):
    """Runs the plotter
    path: complete filepath to a csv
    x: list of strings of the column headings you want as independent variables
    e.g. x = ['Date', 'rzsm']
    y: list of strings of the column headings you want as dependent variables
    e.g. y = ['rain', 'soil_ksat', 'eta', 'rzsm']
    multi = True is default which means the graph makes one graph for each ind var with all dep variables on the same plot
    multi = False means graph will make one ind/dep variable plot for each ind variable and each dep variable.
    eg. three ind vars, three dep vars, 3x3 = 9 graphs total.

    This function makes plots for one csv at a time.
    """

    # now you need to read all the files and return a dict of pandas dataframes.
    dataframe = reader_function(path)
    print 'dataframe', dataframe
    # send in the dict containing the df to a grapher function and get to graphing!
    col_dict = get_columns(dataframe)
    grapher(col_dict, x, y, plot_output, function_count, together=multi)

    print 'complete?!?!'

if __name__ == "__main__":
    run_tracker_plot()

    # TODO find a way to pre-configure what you want to plot, then give x axis stuff (time series) and y axis stuff to grapher.
    # TODO - I must eventually devise a way to use this with the one to one RZSM plots, or make this into an object that can handle something like that.<|endoftext|>


<commit_before>from xml.dom import minidom
import os, sys
import common_tags as tag
from django.core.files.storage import get_storage_class
from django.conf import settings

SLASH = u"/"

class MyError(Exception):
    pass

def image_urls_for_form(xform):
    return sum([
        image_urls(s) for s in xform.surveys.all()
    ], [])

def get_path(path, suffix):
    from os.path import splitext
    fileName, fileExtension = os.path.splitext(path)
    return fileName + suffix +  fileExtension

def image_urls(instance):
    default_storage = get_storage_class()()
    return [ default_storage.url(get_path(a.media_file.name,
            settings.THUMB_CONF['medium']['suffix'])) if
            default_storage.exists(get_path(a.media_file.name,
            settings.THUMB_CONF['medium']['suffix'])) else
            a.media_file.url for a in instance.attachments.all()]

def parse_xform_instance(xml_str):
    """
    'xml_str' is a str object holding the XML of an XForm
    instance. Return a python object representation of this XML file.
    """
    xml_obj = minidom.parseString(xml_str)
    root_node = xml_obj.documentElement
    # go through the xml object creating a corresponding python object
    # NOTE: THIS WILL DESTROY ANY DATA COLLECTED WITH REPEATABLE NODES
    # THIS IS OKAY FOR OUR USE CASE, BUT OTHER USERS SHOULD BEWARE.
    survey_data = dict(_path_value_pairs(root_node))
    assert len(list(_all_attributes(root_node)))==1, \
        u"There should be exactly one attribute in this document."
    survey_data.update({
            tag.XFORM_ID_STRING : root_node.getAttribute(u"id"),
            tag.INSTANCE_DOC_NAME : root_node.nodeName,
            })
    return survey_data

def _path(node):
    n = node
    levels = []
    while n.nodeType!=n.DOCUMENT_NODE:
        levels = [n.nodeName] + levels
        n = n.parentNode
    return SLASH.join(levels[1:])

def _path_value_pairs(node):
    """
    Using a depth first traversal of the xml nodes build up a python
    object in parent that holds the tree structure of the data.
    """
    if len(node.childNodes)==0:
        # there's no data for this leaf node
        yield _path(node), None
    elif len(node.childNodes)==1 and \
            node.childNodes[0].nodeType==node.TEXT_NODE:
        # there is data for this leaf node
        yield _path(node), node.childNodes[0].nodeValue
    else:
        # this is an internal node
        for child in node.childNodes:
            for pair in _path_value_pairs(child):
                yield pair

def _all_attributes(node):
    """
    Go through an XML document returning all the attributes we see.
    """
    if hasattr(node, "hasAttributes") and node.hasAttributes():
        for key in node.attributes.keys():
            yield key, node.getAttribute(key)
    for child in node.childNodes:
        for pair in _all_attributes(child):
            yield pair

# test = {"one/two" : 1, "one/three" : 3, "two" : 2}
# vardict = VariableDictionary(test)
# print vardict["one"]._d, vardict["two"]

class XFormParser(object):
    def __init__(self, xml):
        assert type(xml)==str or type(xml)==unicode, u"xml must be a string"
        self.doc = minidom.parseString(xml)
        self.root_node = self.doc.documentElement

    def get_variable_list(self):
        """
        Return a list of pairs [(path to variable1, attributes of variable1), ...].
        """
        bindings = self.doc.getElementsByTagName(u"bind")
        attributes = [dict(_all_attributes(b)) for b in bindings]
        # note: nodesets look like /water/source/blah we're returning source/blah
        return [(SLASH.join(d.pop(u"nodeset").split(SLASH)[2:]), d) for d in attributes]

    def get_variable_dictionary(self):
        d = {}
        for path, attributes in self.get_variable_list():
            assert path not in d, u"Paths should be unique."
            d[path] = attributes
        return d

    def follow(self, path):
        """
        Path is an array of node names. Starting at the document
        element we follow the path, returning the final node in the
        path.
        """
        element = self.doc.documentElement
        count = {}
        for name in path.split(SLASH):
            count[name] = 0
            for child in element.childNodes:
                if isinstance(child, minidom.Element) and child.tagName==name:
                    count[name] += 1
                    element = child
            assert count[name]==1
        return element

    def get_id_string(self):
        """
        Find the single child of h:head/model/instance and return the
        attribute 'id'.
        """
        instance = self.follow(u"h:head/model/instance")
        children = [child for child in instance.childNodes \
                        if isinstance(child, minidom.Element)]
        assert len(children)==1
        return children[0].getAttribute(u"id")

    def get_title(self):
        title = self.follow(u"h:head/h:title")
        assert len(title.childNodes)==1, u"There should be a single title"
        return title.childNodes[0].nodeValue

    supported_controls = ["input", "select1", "select", "upload"]

    def get_control_dict(self):
        def get_pairs(e):
            result = []
            if hasattr(e, "tagName") and e.tagName in self.supported_controls:
                result.append( (e.getAttribute("ref"),
                                get_text(follow(e, "label").childNodes)) )
            if e.hasChildNodes:
                for child in e.childNodes:
                    result.extend(get_pairs(child))
            return result
        return dict(get_pairs(self.follow("h:body")))

# f = open(sys.argv[1])
# xform = XFormParser(f.read())
# f.close()
# import json ; print json.dumps(xform.get_variable_dictionary(), indent=4)

from django.conf import settings
from django.core.mail import mail_admins
import traceback
def report_exception(subject, info, exc_info=None):
    if exc_info:
        cls, err = exc_info[:2]
        info += u"Exception in request: %s: %s" % (cls.__name__, err)
        info += u"".join(traceback.format_exception(*exc_info))

    if settings.DEBUG:
        print subject
        print info
    else:
        mail_admins(subject=subject, message=info)

from django.core.files.uploadedfile import InMemoryUploadedFile
def django_file(path, field_name, content_type):
    # adapted from here: http://groups.google.com/group/django-users/browse_thread/thread/834f988876ff3c45/
    f = open(path)
    return InMemoryUploadedFile(
        file=f,
        field_name=field_name,
        name=f.name,
        content_type=content_type,
        size=os.path.getsize(path),
        charset=None
        )

<commit_msg>Small import fix<commit_after>from xml.dom import minidom
import os, sys
from os.path import splitext
import common_tags as tag
from django.core.files.storage import get_storage_class
from django.conf import settings

SLASH = u"/"

class MyError(Exception):
    pass

def image_urls_for_form(xform):
    return sum([
        image_urls(s) for s in xform.surveys.all()
    ], [])

def get_path(path, suffix):
    fileName, fileExtension = os.path.splitext(path)
    return fileName + suffix +  fileExtension

def image_urls(instance):
    default_storage = get_storage_class()()
    return [ default_storage.url(get_path(a.media_file.name,
            settings.THUMB_CONF['medium']['suffix'])) if
            default_storage.exists(get_path(a.media_file.name,
            settings.THUMB_CONF['medium']['suffix'])) else
            a.media_file.url for a in instance.attachments.all()]

def parse_xform_instance(xml_str):
    """
    'xml_str' is a str object holding the XML of an XForm
    instance. Return a python object representation of this XML file.
    """
    xml_obj = minidom.parseString(xml_str)
    root_node = xml_obj.documentElement
    # go through the xml object creating a corresponding python object
    # NOTE: THIS WILL DESTROY ANY DATA COLLECTED WITH REPEATABLE NODES
    # THIS IS OKAY FOR OUR USE CASE, BUT OTHER USERS SHOULD BEWARE.
    survey_data = dict(_path_value_pairs(root_node))
    assert len(list(_all_attributes(root_node)))==1, \
        u"There should be exactly one attribute in this document."
    survey_data.update({
            tag.XFORM_ID_STRING : root_node.getAttribute(u"id"),
            tag.INSTANCE_DOC_NAME : root_node.nodeName,
            })
    return survey_data

def _path(node):
    n = node
    levels = []
    while n.nodeType!=n.DOCUMENT_NODE:
        levels = [n.nodeName] + levels
        n = n.parentNode
    return SLASH.join(levels[1:])

def _path_value_pairs(node):
    """
    Using a depth first traversal of the xml nodes build up a python
    object in parent that holds the tree structure of the data.
    """
    if len(node.childNodes)==0:
        # there's no data for this leaf node
        yield _path(node), None
    elif len(node.childNodes)==1 and \
            node.childNodes[0].nodeType==node.TEXT_NODE:
        # there is data for this leaf node
        yield _path(node), node.childNodes[0].nodeValue
    else:
        # this is an internal node
        for child in node.childNodes:
            for pair in _path_value_pairs(child):
                yield pair

def _all_attributes(node):
    """
    Go through an XML document returning all the attributes we see.
    """
    if hasattr(node, "hasAttributes") and node.hasAttributes():
        for key in node.attributes.keys():
            yield key, node.getAttribute(key)
    for child in node.childNodes:
        for pair in _all_attributes(child):
            yield pair

# test = {"one/two" : 1, "one/three" : 3, "two" : 2}
# vardict = VariableDictionary(test)
# print vardict["one"]._d, vardict["two"]

class XFormParser(object):
    def __init__(self, xml):
        assert type(xml)==str or type(xml)==unicode, u"xml must be a string"
        self.doc = minidom.parseString(xml)
        self.root_node = self.doc.documentElement

    def get_variable_list(self):
        """
        Return a list of pairs [(path to variable1, attributes of variable1), ...].
        """
        bindings = self.doc.getElementsByTagName(u"bind")
        attributes = [dict(_all_attributes(b)) for b in bindings]
        # note: nodesets look like /water/source/blah we're returning source/blah
        return [(SLASH.join(d.pop(u"nodeset").split(SLASH)[2:]), d) for d in attributes]

    def get_variable_dictionary(self):
        d = {}
        for path, attributes in self.get_variable_list():
            assert path not in d, u"Paths should be unique."
            d[path] = attributes
        return d

    def follow(self, path):
        """
        Path is an array of node names. Starting at the document
        element we follow the path, returning the final node in the
        path.
        """
        element = self.doc.documentElement
        count = {}
        for name in path.split(SLASH):
            count[name] = 0
            for child in element.childNodes:
                if isinstance(child, minidom.Element) and child.tagName==name:
                    count[name] += 1
                    element = child
            assert count[name]==1
        return element

    def get_id_string(self):
        """
        Find the single child of h:head/model/instance and return the
        attribute 'id'.
        """
        instance = self.follow(u"h:head/model/instance")
        children = [child for child in instance.childNodes \
                        if isinstance(child, minidom.Element)]
        assert len(children)==1
        return children[0].getAttribute(u"id")

    def get_title(self):
        title = self.follow(u"h:head/h:title")
        assert len(title.childNodes)==1, u"There should be a single title"
        return title.childNodes[0].nodeValue

    supported_controls = ["input", "select1", "select", "upload"]

    def get_control_dict(self):
        def get_pairs(e):
            result = []
            if hasattr(e, "tagName") and e.tagName in self.supported_controls:
                result.append( (e.getAttribute("ref"),
                                get_text(follow(e, "label").childNodes)) )
            if e.hasChildNodes:
                for child in e.childNodes:
                    result.extend(get_pairs(child))
            return result
        return dict(get_pairs(self.follow("h:body")))

# f = open(sys.argv[1])
# xform = XFormParser(f.read())
# f.close()
# import json ; print json.dumps(xform.get_variable_dictionary(), indent=4)

from django.conf import settings
from django.core.mail import mail_admins
import traceback
def report_exception(subject, info, exc_info=None):
    if exc_info:
        cls, err = exc_info[:2]
        info += u"Exception in request: %s: %s" % (cls.__name__, err)
        info += u"".join(traceback.format_exception(*exc_info))

    if settings.DEBUG:
        print subject
        print info
    else:
        mail_admins(subject=subject, message=info)

from django.core.files.uploadedfile import InMemoryUploadedFile
def django_file(path, field_name, content_type):
    # adapted from here: http://groups.google.com/group/django-users/browse_thread/thread/834f988876ff3c45/
    f = open(path)
    return InMemoryUploadedFile(
        file=f,
        field_name=field_name,
        name=f.name,
        content_type=content_type,
        size=os.path.getsize(path),
        charset=None
        )

<|endoftext|>


<commit_before>"""
  Abstract models for atmosphere.
  NOTE: These models should NEVER be created directly.
  See the respective sub-classes for complete implementation details.
"""
import uuid
from uuid import uuid4

from django.db import models
from django.db.models import Q
from django.utils import timezone

from core.exceptions import InvalidMembership, ProviderLimitExceeded
from core.query import only_current
from core.models.identity import Identity
from core.models.instance_source import InstanceSource
from core.models.provider import Provider
from core.models.status_type import StatusType, get_status_type_id
from core.models.user import AtmosphereUser as User

UNRESOLVED_STATES = ["pending", "failed"]

class BaseRequest(models.Model):

    """
    Base model which represents a request object
    """
    uuid = models.UUIDField(default=uuid4, unique=True, editable=False)
    status = models.ForeignKey(StatusType)

    # Associated creator and identity
    created_by = models.ForeignKey(User)
    membership = models.ForeignKey("IdentityMembership")

    admin_message = models.CharField(max_length=1024, default="", blank=True)

    # Request Timeline
    start_date = models.DateTimeField(default=timezone.now)
    end_date = models.DateTimeField(null=True, blank=True)

    class Meta:
        abstract = True

    def save(self, *args, **kwargs):
        """
        Only allow one active request per provider
        """
        if not self.pk and self.is_active(self.membership) and self.has_current_requests(self.membership):
            raise ProviderLimitExceeded(
                "The number of open requests has been exceeded.")

        if not self.membership.is_member(self.created_by):
            raise InvalidMembership(
                "This membership does not belong to the user")

        super(BaseRequest, self).save(*args, **kwargs)

    @classmethod
    def has_current_requests(cls, identity_membership):
        """
        Return if the object currently has non end-dated objects
        """
        return cls.objects.filter(only_current()).count() > 0

    @classmethod
    def is_active(cls, identity_membership):
        """
        Return if a request is active for the identity_membership
        """
        return cls.objects.filter(
            membership=identity_membership,
            status__name__in=UNRESOLVED_STATES).count() > 0

    @classmethod
    def get_unresolved(cls):
        """
        Returns all requests that are currently in an unresolved state
        """
        return cls.objects.filter(status__name__in=UNRESOLVED_STATES)

    def is_closed(self):
        return self.status.name not in UNRESOLVED_STATES

    def is_approved(self):
        return self.status.name == "approved"

    def is_denied(self):
        return self.status.name == "denied"

    def can_modify(self, user):
        """
        Returns whether the user can modify the request
        """
        # Only pending requests can be modified by the owner
        if self.created_by.username == user.username:
            return self.status.name == "pending"

        return user.is_staff or user.is_superuser

class BaseSource(models.Model):

    """
    Source object which can be booted
    """
    instance_source = models.OneToOneField(InstanceSource)

    class Meta:
        abstract = True

    def save(self, *args, **kwargs):
        """
        Save the instance_source then the model
        """
        self.instance_source.save()
        super(BaseSource, self).save(*args, **kwargs)

    @property
    def start_date(self):
        return self.instance_source.start_date

    @property
    def end_date(self):
        return self.instance_source.end_date

    @end_date.setter
    def end_date(self, value):
        self.instance_source.end_date = value

    @property
    def provider(self):
        return self.instance_source.provider

    @property
    def identifier(self):
        return self.instance_source.identifier

    def to_dict(self):
        return {
            "start_date": self.start_date,
            "end_date": self.end_date,
            "identifier": self.identifier,
            "provider_uuid": self.provider.uuid
        }

class BaseHistory(models.Model):

    """
    Base model which is used to track changes in another model
    """
    CREATE = "CREATE"
    UPDATE = "UPDATE"
    DELETE = "DELETED"

    OPERATIONS = (
        (CREATE, "The field has been created."),
        (UPDATE, "The field has been updated."),
        (DELETE, "The field has been deleted."),
    )

    field_name = models.CharField(max_length=255)
    operation = models.CharField(max_length=255,
                                 choices=OPERATIONS, default=UPDATE)
    current_value = models.TextField()
    previous_value = models.TextField()
    timestamp = models.DateTimeField(default=timezone.now)

    class Meta:
        abstract = True
<commit_msg>Hack removed during branch trickle-down merge<commit_after>"""
  Abstract models for atmosphere.
  NOTE: These models should NEVER be created directly.
  See the respective sub-classes for complete implementation details.
"""
import uuid
from uuid import uuid4

from django.db import models
from django.db.models import Q
from django.utils import timezone

from core.exceptions import InvalidMembership, ProviderLimitExceeded
from core.query import only_current
from core.models.identity import Identity
from core.models.instance_source import InstanceSource
from core.models.provider import Provider
from core.models.status_type import StatusType, get_status_type_id
from core.models.user import AtmosphereUser as User

UNRESOLVED_STATES = ["pending", "failed"]

class BaseRequest(models.Model):

    """
    Base model which represents a request object
    """
    uuid = models.UUIDField(default=uuid4, unique=True, editable=False)
    status = models.ForeignKey(StatusType)

    # Associated creator and identity
    created_by = models.ForeignKey(User)
    membership = models.ForeignKey("IdentityMembership")

    admin_message = models.CharField(max_length=1024, default="", blank=True)

    # Request Timeline
    start_date = models.DateTimeField(default=timezone.now)
    end_date = models.DateTimeField(null=True, blank=True)

    class Meta:
        abstract = True

    def save(self, *args, **kwargs):
        """
        Only allow one active request per provider
        """
        if not self.pk and self.is_active(self.membership)\
                and self.has_current_requests(self.membership):
            raise ProviderLimitExceeded(
                "The number of open requests has been exceeded.")

        if not self.membership.is_member(self.created_by):
            raise InvalidMembership(
                "This membership does not belong to the user")

        super(BaseRequest, self).save(*args, **kwargs)

    @classmethod
    def has_current_requests(cls, identity_membership):
        """
        Return if the object currently has non end-dated objects
        """
        return cls.objects.filter(only_current()).count() > 0

    @classmethod
    def is_active(cls, identity_membership):
        """
        Return if a request is active for the identity_membership
        """
        return cls.objects.filter(
            membership=identity_membership,
            status__name__in=UNRESOLVED_STATES).count() > 0

    @classmethod
    def get_unresolved(cls):
        """
        Returns all requests that are currently in an unresolved state
        """
        return cls.objects.filter(status__name__in=UNRESOLVED_STATES)

    def is_closed(self):
        return self.status.name not in UNRESOLVED_STATES

    def is_approved(self):
        return self.status.name == "approved"

    def is_denied(self):
        return self.status.name == "denied"

    def can_modify(self, user):
        """
        Returns whether the user can modify the request
        """
        # Only pending requests can be modified by the owner
        if self.created_by.username == user.username:
            return self.status.name == "pending"

        return user.is_staff or user.is_superuser

class BaseSource(models.Model):

    """
    Source object which can be booted
    """
    instance_source = models.OneToOneField(InstanceSource)

    class Meta:
        abstract = True

    def save(self, *args, **kwargs):
        """
        Save the instance_source then the model
        """
        self.instance_source.save()
        super(BaseSource, self).save(*args, **kwargs)

    @property
    def start_date(self):
        return self.instance_source.start_date

    @property
    def end_date(self):
        return self.instance_source.end_date

    @end_date.setter
    def end_date(self, value):
        self.instance_source.end_date = value

    @property
    def provider(self):
        return self.instance_source.provider

    @property
    def identifier(self):
        return self.instance_source.identifier

    def to_dict(self):
        return {
            "start_date": self.start_date,
            "end_date": self.end_date,
            "identifier": self.identifier,
            "provider_uuid": self.provider.uuid
        }

class BaseHistory(models.Model):

    """
    Base model which is used to track changes in another model
    """
    CREATE = "CREATE"
    UPDATE = "UPDATE"
    DELETE = "DELETED"

    OPERATIONS = (
        (CREATE, "The field has been created."),
        (UPDATE, "The field has been updated."),
        (DELETE, "The field has been deleted."),
    )

    field_name = models.CharField(max_length=255)
    operation = models.CharField(max_length=255,
                                 choices=OPERATIONS, default=UPDATE)
    current_value = models.TextField()
    previous_value = models.TextField()
    timestamp = models.DateTimeField(default=timezone.now)

    class Meta:
        abstract = True
<|endoftext|>