<commit_before>import sublime, sublime_plugin
import desktop
import tempfile
import markdown2
import os
import sys
import re
import json
import urllib2

settings = sublime.load_settings('MarkdownPreview.sublime-settings')

def getTempMarkdownPreviewPath(view):
    " return a permanent full path of the temp markdown preview file "
    tmp_filename = '%s.html' % view.id()
    tmp_fullpath = os.path.join(tempfile.gettempdir(), tmp_filename)
    return tmp_fullpath

class MarkdownPreviewListener(sublime_plugin.EventListener):
    """ update the output html when markdown file has already been converted once """

    def on_post_save(self, view):
        if view.file_name().endswith(tuple(settings.get('markdown_filetypes', (".md", ".markdown", ".mdown")))):
            temp_file = getTempMarkdownPreviewPath(view)
            if os.path.isfile(temp_file):
                # reexec markdown conversion
                view.run_command('markdown_preview', {'target': 'disk'})
                sublime.status_message('Markdown preview file updated')

class MarkdownCheatsheetCommand(sublime_plugin.TextCommand):
    """ open our markdown cheat sheet in ST2"""
    def run(self, edit):
        cheatsheet = os.path.join(sublime.packages_path(), 'Markdown Preview', 'sample.md')
        self.view.window().open_file(cheatsheet)
        sublime.status_message('Markdown cheat sheet opened')

class MarkdownPreviewCommand(sublime_plugin.TextCommand):
    """ preview file contents with python-markdown and your web browser"""

    def getCSS(self):
        config_parser = settings.get('parser')
        config_css = settings.get('css')

        if config_css and config_css != 'default':
            return "<link href='%s' rel='stylesheet' type='text/css'>" % config_css
        else:
            css_filename = 'markdown.css'
            if config_parser and config_parser == 'github':
                css_filename = 'github.css'
            # path via package manager
            css_path = os.path.join(sublime.packages_path(), 'Markdown Preview', css_filename)
            if not os.path.isfile(css_path):
                # path via git repo
                css_path = os.path.join(sublime.packages_path(), 'sublimetext-markdown-preview', css_filename)
                if not os.path.isfile(css_path):
                    sublime.error_message('markdown.css file not found!')
                    raise Exception("markdown.css file not found!")
            styles = u"" % open(css_path, 'r').read().decode('utf-8')
        return styles

    def postprocessor(self, html):
        # fix relative paths in images/scripts
        def tag_fix(match):
            tag, src = match.groups()
            filename = self.view.file_name()
            if filename:
                if not src.startswith(('file://', 'https://', 'http://', '/')):
                    abs_path = u'file://%s/%s' % (os.path.dirname(filename), src)
                    tag = tag.replace(src, abs_path)
            return tag
        RE_SOURCES = re.compile("""(?P<tag><(?:img|script)[^>]+src=["'](?P<src>[^"']+)[^>]*>)""")
        html = RE_SOURCES.sub(tag_fix, html)
        return html

    def run(self, edit, target='browser'):
        region = sublime.Region(0, self.view.size())
        encoding = self.view.encoding()
        if encoding == 'Undefined':
            encoding = 'utf-8'
        elif encoding == 'Western (Windows 1252)':
            encoding = 'windows-1252'
        contents = self.view.substr(region)

        config_parser = settings.get('parser')

        markdown_html = u'cannot convert markdown'
        if config_parser and config_parser == 'github':
            sublime.status_message('converting markdown with github API...')
            try:
                #contents = contents.replace('%', '')    # see https://gist.github.com/3742011
                data = json.dumps({"text": contents, "mode": "gfm"})
                url = "https://api.github.com/markdown"
                request = urllib2.Request(url, data, {'Content-Type': 'application/json'})
                markdown_html = urllib2.urlopen(request).read().decode('utf-8')
            except urllib2.HTTPError:
                sublime.error_message('github API responded in an unfashion way :/')
            except urllib2.URLError:
                sublime.error_message('cannot use github API to convert markdown. SSL is not included in your Python installation')
            except:
                sublime.error_message('cannot use github API to convert markdown. Please check your settings.')
            else:
                sublime.status_message('converted markdown with github API successfully')
        else:
            # convert the markdown
            markdown_html = markdown2.markdown(contents, extras=['footnotes', 'toc', 'fenced-code-blocks', 'cuddled-lists'])
            toc_html = markdown_html.toc_html
            if toc_html:
                toc_markers = ['[toc]', '[TOC]', '']
                for marker in toc_markers:
                    markdown_html = markdown_html.replace(marker, toc_html)

            # postprocess the html
            markdown_html = self.postprocessor(markdown_html)

        # check if LiveReload ST2 extension installed
        livereload_installed = ('LiveReload' in os.listdir(sublime.packages_path()))

        if target in ['disk', 'browser']:
            # build the html
            html_contents = u'<!DOCTYPE html>'
            html_contents += '<meta charset="%s">' % encoding
            html_contents += self.getCSS()
            if livereload_installed:
                html_contents += ''
            html_contents += ''
            html_contents += markdown_html
            html_contents += ''

            # update output html file
            tmp_fullpath = getTempMarkdownPreviewPath(self.view)
            tmp_html = open(tmp_fullpath, 'w')
            tmp_html.write(html_contents.encode(encoding))
            tmp_html.close()
            # todo : livereload ?
            if target == 'browser':
                config_browser = settings.get('browser')
                if config_browser and config_browser != 'default':
                    cmd = '"%s" %s' % (config_browser, tmp_fullpath)
                    if sys.platform == 'darwin':
                        cmd = "open -a %s" % cmd
                    print "Markdown Preview: executing", cmd
                    result = os.system(cmd)
                    if result != 0:
                        sublime.error_message('cannot execute "%s" Please check your Markdown Preview settings' % config_browser)
                    else:
                        sublime.status_message('Markdown preview launched in %s' % config_browser)
                else:
                    desktop.open(tmp_fullpath)
                    sublime.status_message('Markdown preview launched in default html viewer')
        elif target == 'sublime':
            # build the html
            html_contents = markdown_html
            new_view = self.view.window().new_file()
            new_view.set_scratch(True)
            new_edit = new_view.begin_edit()
            new_view.insert(new_edit, 0, html_contents)
            new_view.end_edit(new_edit)
            sublime.status_message('Markdown preview launched in sublime')
<commit_msg>Add support for UTF-8 with BOM encoding<commit_after>import sublime, sublime_plugin
import desktop
import tempfile
import markdown2
import os
import sys
import re
import json
import urllib2

settings = sublime.load_settings('MarkdownPreview.sublime-settings')

def getTempMarkdownPreviewPath(view):
    " return a permanent full path of the temp markdown preview file "
    tmp_filename = '%s.html' % view.id()
    tmp_fullpath = os.path.join(tempfile.gettempdir(), tmp_filename)
    return tmp_fullpath

class MarkdownPreviewListener(sublime_plugin.EventListener):
    """ update the output html when markdown file has already been converted once """

    def on_post_save(self, view):
        if view.file_name().endswith(tuple(settings.get('markdown_filetypes', (".md", ".markdown", ".mdown")))):
            temp_file = getTempMarkdownPreviewPath(view)
            if os.path.isfile(temp_file):
                # reexec markdown conversion
                view.run_command('markdown_preview', {'target': 'disk'})
                sublime.status_message('Markdown preview file updated')

class MarkdownCheatsheetCommand(sublime_plugin.TextCommand):
    """ open our markdown cheat sheet in ST2"""
    def run(self, edit):
        cheatsheet = os.path.join(sublime.packages_path(), 'Markdown Preview', 'sample.md')
        self.view.window().open_file(cheatsheet)
        sublime.status_message('Markdown cheat sheet opened')

class MarkdownPreviewCommand(sublime_plugin.TextCommand):
    """ preview file contents with python-markdown and your web browser"""

    def getCSS(self):
        config_parser = settings.get('parser')
        config_css = settings.get('css')

        if config_css and config_css != 'default':
            return "<link href='%s' rel='stylesheet' type='text/css'>" % config_css
        else:
            css_filename = 'markdown.css'
            if config_parser and config_parser == 'github':
                css_filename = 'github.css'
            # path via package manager
            css_path = os.path.join(sublime.packages_path(), 'Markdown Preview', css_filename)
            if not os.path.isfile(css_path):
                # path via git repo
                css_path = os.path.join(sublime.packages_path(), 'sublimetext-markdown-preview', css_filename)
                if not os.path.isfile(css_path):
                    sublime.error_message('markdown.css file not found!')
                    raise Exception("markdown.css file not found!")
            styles = u"" % open(css_path, 'r').read().decode('utf-8')
        return styles

    def postprocessor(self, html):
        # fix relative paths in images/scripts
        def tag_fix(match):
            tag, src = match.groups()
            filename = self.view.file_name()
            if filename:
                if not src.startswith(('file://', 'https://', 'http://', '/')):
                    abs_path = u'file://%s/%s' % (os.path.dirname(filename), src)
                    tag = tag.replace(src, abs_path)
            return tag
        RE_SOURCES = re.compile("""(?P<tag><(?:img|script)[^>]+src=["'](?P<src>[^"']+)[^>]*>)""")
        html = RE_SOURCES.sub(tag_fix, html)
        return html

    def run(self, edit, target='browser'):
        region = sublime.Region(0, self.view.size())
        encoding = self.view.encoding()
        if encoding == 'Undefined':
            encoding = 'utf-8'
        elif encoding == 'Western (Windows 1252)':
            encoding = 'windows-1252'
        elif encoding == 'UTF-8 with BOM':
            encoding = 'utf-8'
        contents = self.view.substr(region)

        config_parser = settings.get('parser')

        markdown_html = u'cannot convert markdown'
        if config_parser and config_parser == 'github':
            sublime.status_message('converting markdown with github API...')
            try:
                #contents = contents.replace('%', '')    # see https://gist.github.com/3742011
                data = json.dumps({"text": contents, "mode": "gfm"})
                url = "https://api.github.com/markdown"
                request = urllib2.Request(url, data, {'Content-Type': 'application/json'})
                markdown_html = urllib2.urlopen(request).read().decode('utf-8')
            except urllib2.HTTPError:
                sublime.error_message('github API responded in an unfashion way :/')
            except urllib2.URLError:
                sublime.error_message('cannot use github API to convert markdown. SSL is not included in your Python installation')
            except:
                sublime.error_message('cannot use github API to convert markdown. Please check your settings.')
            else:
                sublime.status_message('converted markdown with github API successfully')
        else:
            # convert the markdown
            markdown_html = markdown2.markdown(contents, extras=['footnotes', 'toc', 'fenced-code-blocks', 'cuddled-lists'])
            toc_html = markdown_html.toc_html
            if toc_html:
                toc_markers = ['[toc]', '[TOC]', '']
                for marker in toc_markers:
                    markdown_html = markdown_html.replace(marker, toc_html)

            # postprocess the html
            markdown_html = self.postprocessor(markdown_html)

        # check if LiveReload ST2 extension installed
        livereload_installed = ('LiveReload' in os.listdir(sublime.packages_path()))

        if target in ['disk', 'browser']:
            # build the html
            html_contents = u'<!DOCTYPE html>'
            html_contents += '<meta charset="%s">' % encoding
            html_contents += self.getCSS()
            if livereload_installed:
                html_contents += ''
            html_contents += ''
            html_contents += markdown_html
            html_contents += ''

            # update output html file
            tmp_fullpath = getTempMarkdownPreviewPath(self.view)
            tmp_html = open(tmp_fullpath, 'w')
            tmp_html.write(html_contents.encode(encoding))
            tmp_html.close()
            # todo : livereload ?
            if target == 'browser':
                config_browser = settings.get('browser')
                if config_browser and config_browser != 'default':
                    cmd = '"%s" %s' % (config_browser, tmp_fullpath)
                    if sys.platform == 'darwin':
                        cmd = "open -a %s" % cmd
                    print "Markdown Preview: executing", cmd
                    result = os.system(cmd)
                    if result != 0:
                        sublime.error_message('cannot execute "%s" Please check your Markdown Preview settings' % config_browser)
                    else:
                        sublime.status_message('Markdown preview launched in %s' % config_browser)
                else:
                    desktop.open(tmp_fullpath)
                    sublime.status_message('Markdown preview launched in default html viewer')
        elif target == 'sublime':
            # build the html
            html_contents = markdown_html
            new_view = self.view.window().new_file()
            new_view.set_scratch(True)
            new_edit = new_view.begin_edit()
            new_view.insert(new_edit, 0, html_contents)
            new_view.end_edit(new_edit)
            sublime.status_message('Markdown preview launched in sublime')
<|endoftext|>


<commit_before>"""OANDA API wrapper for OANDA's REST-V20 API."""

import json
import requests
from .exceptions import V20Error

TRADING_ENVIRONMENTS = {
    "practice": {
        "stream": 'https://stream-fxpractice.oanda.com',
        "api": 'https://api-fxpractice.oanda.com'
    },
    "live": {
        "stream": 'https://stream-fxtrade.oanda.com',
        "api": 'https://api-fxtrade.oanda.com'
    }
}

class API(object):
    r"""API - class to handle APIRequests objects to access API endpoints.

    Examples
    --------

    ::

        # get a list of trades
        from oandapyV20 import API
        import oandapyV20.endpoints.trades as trades

        api = API(access_token="xxx")
        accountID = "101-305-3091856-001"

        r = trades.Trades(accountID, op=trades.TRADE_LIST)
        # show the endpoint as it is constructed for this call
        print("REQUEST:{}".format(r))
        rv = api.request(r)
        print("RESPONSE:\n{}".format(json.dumps(rv, indent=2)))

    Output::

        REQUEST:v3/accounts/101-305-3091856-001/trades
        RESPONSE:
        "trades": [
            {
              "financing": "0.0000",
              "openTime": "2016-07-21T15:47:05.170212014Z",
              "price": "10133.9",
              "unrealizedPL": "8.0000",
              "realizedPL": "0.0000",
              "instrument": "DE30_EUR",
              "state": "OPEN",
              "initialUnits": "-10",
              "currentUnits": "-10",
              "id": "1032"
            },
            {
              "financing": "0.0000",
              "openTime": "2016-07-21T15:47:04.963590941Z",
              "price": "10134.4",
              "unrealizedPL": "13.0000",
              "realizedPL": "0.0000",
              "instrument": "DE30_EUR",
              "state": "OPEN",
              "initialUnits": "-10",
              "currentUnits": "-10",
              "id": "1030"
            }
          ],
          "lastTransactionID": "1040"
        }

    ::

        # reduce a trade by it's id
        from oandapyV20 import API
        import oandapyV20.endpoints.trades as trades

        headers = {"Content-Type": "application/json"}
        api = API(access_token="...", headers=headers)

        accountID = "101-305-3091856-001"
        tradeID = "1030"
        cfg = { "units": 5 }
        r = trades.Trades(accountID, tradeID=tradeID,
                          op=trades.TRADE_CLOSE, data=cfg)
        # show the endpoint as it is constructed for this call
        print("REQUEST:{}".format(r))
        rv = api.request(r)
        print("RESPONSE\n{}".format(json.dumps(rv, indent=2)))

    Output::

        REQUEST:v3/accounts/101-305-3091856-001/trades/1030/close
        RESPONSE: {
          "orderFillTransaction": {
            "orderID": "1041",
            "financing": "-0.1519",
            "instrument": "DE30_EUR",
            "userID": 1435156,
            "price": "10131.6",
            "tradeReduced": {
              "units": "5",
              "financing": "-0.1519",
              "realizedPL": "14.0000",
              "tradeID": "1030"
            },
            "batchID": "1041",
            "accountBalance": "44876.2548",
            "reason": "MARKET_ORDER_TRADE_CLOSE",
            "time": "2016-07-21T17:32:51.361464739Z",
            "units": "5",
            "type": "ORDER_FILL",
            "id": "1042",
            "pl": "14.0000",
            "accountID": "101-305-3091856-001"
          },
          "orderCreateTransaction": {
            "timeInForce": "FOK",
            "positionFill": "REDUCE_ONLY",
            "userID": 1435156,
            "batchID": "1041",
            "instrument": "DE30_EUR",
            "reason": "TRADE_CLOSE",
            "tradeClose": {
              "units": "5",
              "tradeID": "1030"
            },
            "time": "2016-07-21T17:32:51.361464739Z",
            "units": "5",
            "type": "MARKET_ORDER",
            "id": "1041",
            "accountID": "101-305-3091856-001"
          },
          "relatedTransactionIDs": [
            "1041",
            "1042"
          ],
          "lastTransactionID": "1042"
        }
    """

    def __init__(self, access_token, environment="practice", headers=None):
        """Instantiate an instance of OandaPy's API wrapper.

        Parameters
        ----------
        access_token : string
            Provide a valid access token.

        environment : string
            Provide the environment for OANDA's REST api. Valid values:
            'practice' or 'live'. Default: 'practice'.

        headers : dict (optional)
            Provide request headers to be set for a request. Several API
            endpoints need data in a JSON format. These calls require the
            header: 'Content-Type: application/json'.

        """
        try:
            TRADING_ENVIRONMENTS[environment]
        except:
            raise KeyError("Unknown environment: {}".format(environment))
        else:
            self.environment = environment

        self.access_token = access_token
        self.client = requests.Session()
        self._connected = False
        self.client.stream = False

        # personal token authentication
        if self.access_token:
            self.client.headers['Authorization'] = 'Bearer '+self.access_token

        if headers:
            self.client.headers.update(headers)

    def disconnect(self):
        """disconnect.

        disconnect a streaming connection. The __stream_request generator
        wil terminate.
        """
        self._connected = False

    def __request(self, method, url, request_args):
        func = getattr(self.client, method)

        response = None
        try:
            response = func(url, **request_args)
        except requests.RequestException as e:
            # log it ?
            raise e
        else:
            self._connected = True

        # Handle error responses
        if response.status_code >= 400:
            raise V20Error(response.status_code,
                           response.content.decode('utf-8'))
        return response

    def __api_request(self, method, url, request_args):
        """_api_request.

        make a 'regular' request. This method is called by
        the 'request' method after it has determined which
        call applies: regular or streaming.
        """
        response = self._request(method, url, request_args)
        content = response.content.decode('utf-8')
        content = json.loads(content)

        return content

    def __stream_request(self, method, url, request_args):
        """_stream_request.

        make a 'stream' request. This method is called by
        the 'request' method after it has determined which
        call applies: regular or streaming.
        """
        response = self._request(method, url, request_args)
        for line in response.iter_lines(90):
            if not self._connected:
                break

            if line:
                data = json.loads(line.decode("utf-8"))
                yield data

    def request(self, endpoint):
        """Perform a request for the APIRequest instance 'endpoint'.

        Parameters
        ----------
        endpoint : APIRequest
            The endpoint parameter contains an instance of an APIRequest
            containing the endpoint, method and optionally other parameters
            or body data.

        Raises
        ------
            V20Error in case of HTTP response code >= 400
        """
        at = "api"
        if hasattr(endpoint, "STREAM") and getattr(endpoint, "STREAM") is True:
            self.client.stream = True
            at = "stream"

        url = "{}/{}".format(TRADING_ENVIRONMENTS[self.environment][at],
                             endpoint)

        method = endpoint.method
        method = method.lower()
        params = None
        try:
            params = getattr(endpoint, "params")
        except AttributeError:
            # request does not have params
            params = {}

        request_args = {}
        if method == 'get':
            request_args['params'] = params
        elif hasattr(endpoint, "data") and endpoint.data:
            request_args['data'] = json.dumps(endpoint.data)

        if at == "api":
            content = self._api_request(method, url, request_args)
            endpoint.response(content)
            return content
        else:
            return self._stream_request(method, url, request_args)
<commit_msg>and the ones mist in the previous commit: _ to __<commit_after>"""OANDA API wrapper for OANDA's REST-V20 API."""

import json
import requests
from .exceptions import V20Error

TRADING_ENVIRONMENTS = {
    "practice": {
        "stream": 'https://stream-fxpractice.oanda.com',
        "api": 'https://api-fxpractice.oanda.com'
    },
    "live": {
        "stream": 'https://stream-fxtrade.oanda.com',
        "api": 'https://api-fxtrade.oanda.com'
    }
}

class API(object):
    r"""API - class to handle APIRequests objects to access API endpoints.

    Examples
    --------

    ::

        # get a list of trades
        from oandapyV20 import API
        import oandapyV20.endpoints.trades as trades

        api = API(access_token="xxx")
        accountID = "101-305-3091856-001"

        r = trades.Trades(accountID, op=trades.TRADE_LIST)
        # show the endpoint as it is constructed for this call
        print("REQUEST:{}".format(r))
        rv = api.request(r)
        print("RESPONSE:\n{}".format(json.dumps(rv, indent=2)))

    Output::

        REQUEST:v3/accounts/101-305-3091856-001/trades
        RESPONSE:
        "trades": [
            {
              "financing": "0.0000",
              "openTime": "2016-07-21T15:47:05.170212014Z",
              "price": "10133.9",
              "unrealizedPL": "8.0000",
              "realizedPL": "0.0000",
              "instrument": "DE30_EUR",
              "state": "OPEN",
              "initialUnits": "-10",
              "currentUnits": "-10",
              "id": "1032"
            },
            {
              "financing": "0.0000",
              "openTime": "2016-07-21T15:47:04.963590941Z",
              "price": "10134.4",
              "unrealizedPL": "13.0000",
              "realizedPL": "0.0000",
              "instrument": "DE30_EUR",
              "state": "OPEN",
              "initialUnits": "-10",
              "currentUnits": "-10",
              "id": "1030"
            }
          ],
          "lastTransactionID": "1040"
        }

    ::

        # reduce a trade by it's id
        from oandapyV20 import API
        import oandapyV20.endpoints.trades as trades

        headers = {"Content-Type": "application/json"}
        api = API(access_token="...", headers=headers)

        accountID = "101-305-3091856-001"
        tradeID = "1030"
        cfg = { "units": 5 }
        r = trades.Trades(accountID, tradeID=tradeID,
                          op=trades.TRADE_CLOSE, data=cfg)
        # show the endpoint as it is constructed for this call
        print("REQUEST:{}".format(r))
        rv = api.request(r)
        print("RESPONSE\n{}".format(json.dumps(rv, indent=2)))

    Output::

        REQUEST:v3/accounts/101-305-3091856-001/trades/1030/close
        RESPONSE: {
          "orderFillTransaction": {
            "orderID": "1041",
            "financing": "-0.1519",
            "instrument": "DE30_EUR",
            "userID": 1435156,
            "price": "10131.6",
            "tradeReduced": {
              "units": "5",
              "financing": "-0.1519",
              "realizedPL": "14.0000",
              "tradeID": "1030"
            },
            "batchID": "1041",
            "accountBalance": "44876.2548",
            "reason": "MARKET_ORDER_TRADE_CLOSE",
            "time": "2016-07-21T17:32:51.361464739Z",
            "units": "5",
            "type": "ORDER_FILL",
            "id": "1042",
            "pl": "14.0000",
            "accountID": "101-305-3091856-001"
          },
          "orderCreateTransaction": {
            "timeInForce": "FOK",
            "positionFill": "REDUCE_ONLY",
            "userID": 1435156,
            "batchID": "1041",
            "instrument": "DE30_EUR",
            "reason": "TRADE_CLOSE",
            "tradeClose": {
              "units": "5",
              "tradeID": "1030"
            },
            "time": "2016-07-21T17:32:51.361464739Z",
            "units": "5",
            "type": "MARKET_ORDER",
            "id": "1041",
            "accountID": "101-305-3091856-001"
          },
          "relatedTransactionIDs": [
            "1041",
            "1042"
          ],
          "lastTransactionID": "1042"
        }
    """

    def __init__(self, access_token, environment="practice", headers=None):
        """Instantiate an instance of OandaPy's API wrapper.

        Parameters
        ----------
        access_token : string
            Provide a valid access token.

        environment : string
            Provide the environment for OANDA's REST api. Valid values:
            'practice' or 'live'. Default: 'practice'.

        headers : dict (optional)
            Provide request headers to be set for a request. Several API
            endpoints need data in a JSON format. These calls require the
            header: 'Content-Type: application/json'.

        """
        try:
            TRADING_ENVIRONMENTS[environment]
        except:
            raise KeyError("Unknown environment: {}".format(environment))
        else:
            self.environment = environment

        self.access_token = access_token
        self.client = requests.Session()
        self._connected = False
        self.client.stream = False

        # personal token authentication
        if self.access_token:
            self.client.headers['Authorization'] = 'Bearer '+self.access_token

        if headers:
            self.client.headers.update(headers)

    def disconnect(self):
        """disconnect.

        disconnect a streaming connection. The __stream_request generator
        wil terminate.
        """
        self._connected = False

    def __request(self, method, url, request_args):
        func = getattr(self.client, method)

        response = None
        try:
            response = func(url, **request_args)
        except requests.RequestException as e:
            # log it ?
            raise e
        else:
            self._connected = True

        # Handle error responses
        if response.status_code >= 400:
            raise V20Error(response.status_code,
                           response.content.decode('utf-8'))
        return response

    def __api_request(self, method, url, request_args):
        """_api_request.

        make a 'regular' request. This method is called by
        the 'request' method after it has determined which
        call applies: regular or streaming.
        """
        response = self.__request(method, url, request_args)
        content = response.content.decode('utf-8')
        content = json.loads(content)

        return content

    def __stream_request(self, method, url, request_args):
        """_stream_request.

        make a 'stream' request. This method is called by
        the 'request' method after it has determined which
        call applies: regular or streaming.
        """
        response = self.__request(method, url, request_args)
        for line in response.iter_lines(90):
            if not self._connected:
                break

            if line:
                data = json.loads(line.decode("utf-8"))
                yield data

    def request(self, endpoint):
        """Perform a request for the APIRequest instance 'endpoint'.

        Parameters
        ----------
        endpoint : APIRequest
            The endpoint parameter contains an instance of an APIRequest
            containing the endpoint, method and optionally other parameters
            or body data.

        Raises
        ------
            V20Error in case of HTTP response code >= 400
        """
        at = "api"
        if hasattr(endpoint, "STREAM") and getattr(endpoint, "STREAM") is True:
            self.client.stream = True
            at = "stream"

        url = "{}/{}".format(TRADING_ENVIRONMENTS[self.environment][at],
                             endpoint)

        method = endpoint.method
        method = method.lower()
        params = None
        try:
            params = getattr(endpoint, "params")
        except AttributeError:
            # request does not have params
            params = {}

        request_args = {}
        if method == 'get':
            request_args['params'] = params
        elif hasattr(endpoint, "data") and endpoint.data:
            request_args['data'] = json.dumps(endpoint.data)

        if at == "api":
            content = self.__api_request(method, url, request_args)
            endpoint.response(content)
            return content
        else:
            return self.__stream_request(method, url, request_args)
<|endoftext|>


<commit_before>"""OANDA API wrapper for OANDA's REST-V20 API."""

import json
import requests
from .exceptions import V20Error

TRADING_ENVIRONMENTS = {
    "practice": 'https://api-fxpractice.oanda.com',
    "live": 'https://api-fxtrade.oanda.com'
}

class API(object):
    """API - class to handle requests to access API endpoints."""

    def __init__(self, access_token, environment="practice", headers=None):
        """Instantiate an instance of OandaPy's API wrapper.

        Parameters
        ----------
        access_token : string
            Provide a valid access token.

        environment : string
            Provide the environment for OANDA's REST api. Valid values:
            'practice' or 'live'. Default: 'practice'.

        headers : dict (optional)
            Provide request headers to be set for a request. Several API
            endpoints need data in a JSON format. These calls require the
            header: 'Content-Type: application/json'.
        """
        try:
            self.api_url = TRADING_ENVIRONMENTS[environment]
        except:
            raise KeyError("Unknown environment: {}".format(environment))

        self.access_token = access_token
        self.client = requests.Session()

        # personal token authentication
        if self.access_token:
            self.client.headers['Authorization'] = 'Bearer '+self.access_token

        if headers:
            self.client.headers.update(headers)

    def request(self, endpoint, params=None):
        """Perform a request for the APIRequest instance 'endpoint'.

        Parameters
        ----------
        endpoint : APIRequest
            The endpoint parameter contains an instance of an APIRequest
            containing the endpoint, method and optionally other parameters.

        params : dict
            The params paremeter optionally provides a dictionary with
            parameters for the request. Data for POST, PUT and PATCH is
            passed this way.
        """
        url = "{}/{}".format(self.api_url, endpoint)

        method = endpoint.method
        method = method.lower()
        params = params or {}

        func = getattr(self.client, method)

        request_args = {}
        if method == 'get':
            request_args['params'] = params
        elif endpoint.body:
            request_args['data'] = json.dumps(endpoint.body)

        response = None
        try:
            response = func(url, **request_args)
        except requests.RequestException as e:
            print (str(e))
        content = response.content.decode('utf-8')

        # Handle error responses
        if response.status_code >= 400:
            raise V20Error(response.status_code, content)

        content = json.loads(content)

        endpoint.response(content)
        return content
<commit_msg>re raise exception in case of an requests.RequestException<commit_after>"""OANDA API wrapper for OANDA's REST-V20 API."""

import json
import requests
from .exceptions import V20Error

TRADING_ENVIRONMENTS = {
    "practice": 'https://api-fxpractice.oanda.com',
    "live": 'https://api-fxtrade.oanda.com'
}

class API(object):
    """API - class to handle requests to access API endpoints."""

    def __init__(self, access_token, environment="practice", headers=None):
        """Instantiate an instance of OandaPy's API wrapper.

        Parameters
        ----------
        access_token : string
            Provide a valid access token.

        environment : string
            Provide the environment for OANDA's REST api. Valid values:
            'practice' or 'live'. Default: 'practice'.

        headers : dict (optional)
            Provide request headers to be set for a request. Several API
            endpoints need data in a JSON format. These calls require the
            header: 'Content-Type: application/json'.
        """
        try:
            self.api_url = TRADING_ENVIRONMENTS[environment]
        except:
            raise KeyError("Unknown environment: {}".format(environment))

        self.access_token = access_token
        self.client = requests.Session()

        # personal token authentication
        if self.access_token:
            self.client.headers['Authorization'] = 'Bearer '+self.access_token

        if headers:
            self.client.headers.update(headers)

    def request(self, endpoint, params=None):
        """Perform a request for the APIRequest instance 'endpoint'.

        Parameters
        ----------
        endpoint : APIRequest
            The endpoint parameter contains an instance of an APIRequest
            containing the endpoint, method and optionally other parameters.

        params : dict
            The params paremeter optionally provides a dictionary with
            parameters for the request. Data for POST, PUT and PATCH is
            passed this way.
        """
        url = "{}/{}".format(self.api_url, endpoint)

        method = endpoint.method
        method = method.lower()
        params = params or {}

        func = getattr(self.client, method)

        request_args = {}
        if method == 'get':
            request_args['params'] = params
        elif endpoint.body:
            request_args['data'] = json.dumps(endpoint.body)

        response = None
        try:
            response = func(url, **request_args)
        except requests.RequestException as e:
            # log it ?
            raise e

        content = response.content.decode('utf-8')

        # Handle error responses
        if response.status_code >= 400:
            raise V20Error(response.status_code, content)

        content = json.loads(content)

        endpoint.response(content)
        return content
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

class Observable:
    """event system for python"""
    class EventNotFound(Exception):
        """Raised if an event was not found"""
        def __init__(self, event):
            Exception.__init__(self, "Event '%s' was not found" % event)

    class HandlerNotFound(Exception):
        """Raised if an handler was not found"""
        def __init__(self, event, handler):
            Exception.__init__(self, "Handler '%s' was not found for event '%s'" % (event, handler))

    def __init__(self):
        self._events = {}

    def on(self, event, func=None):
        """register a handler to a specified event"""
        def _on_wrapper(func):
            if event not in self._events:
                self._events[event] = []
            self._events[event].append(func)
            return func

        if func:
            return _on_wrapper(func)
        else:
            return _on_wrapper

    def once(self, event, func=None):
        """register a handler to a specified event, but remove it when it is triggered"""
        def _once_wrapper(func):
            def _wrapper(*args, **kwargs):
                func(*args, **kwargs)
                self.off(event, _wrapper)
            return _wrapper

        if func:
            return self.on(event, _once_wrapper(func))
        else:
            return lambda func: self.on(event, _once_wrapper(func))

    def off(self, event=None, func=None):
        """unregister an event or handler from an event"""
        if not event:
            self._events = {}
            return True
        if event not in self._events:
            raise Observable.EventNotFound(event)
        if not func:
            self._events[event] = []
            return True
        if not isinstance(func, list):
            func = [func]
        for f in func:
            if f not in self._events[event]:
                raise Observable.HanlderNotFound(event, f.func_name)
            self._events[event].remove(f)

    def trigger(self, event, *args, **kwargs):
        """trigger all functions from an event"""
        if event not in self._events:
            raise Observable.EventNotFound(event)

        handled = False
        for func in self._events[event]:
            func(*args, **kwargs)
            handled = True
        return handled
<commit_msg>corrected spelling mistake<commit_after># -*- coding: utf-8 -*-

class Observable:
    """event system for python"""
    class EventNotFound(Exception):
        """Raised if an event was not found"""
        def __init__(self, event):
            Exception.__init__(self, "Event '%s' was not found" % event)

    class HandlerNotFound(Exception):
        """Raised if an handler was not found"""
        def __init__(self, event, handler):
            Exception.__init__(self, "Handler '%s' was not found for event '%s'" % (event, handler))

    def __init__(self):
        self._events = {}

    def on(self, event, func=None):
        """register a handler to a specified event"""
        def _on_wrapper(func):
            if event not in self._events:
                self._events[event] = []
            self._events[event].append(func)
            return func

        if func:
            return _on_wrapper(func)
        else:
            return _on_wrapper

    def once(self, event, func=None):
        """register a handler to a specified event, but remove it when it is triggered"""
        def _once_wrapper(func):
            def _wrapper(*args, **kwargs):
                func(*args, **kwargs)
                self.off(event, _wrapper)
            return _wrapper

        if func:
            return self.on(event, _once_wrapper(func))
        else:
            return lambda func: self.on(event, _once_wrapper(func))

    def off(self, event=None, func=None):
        """unregister an event or handler from an event"""
        if not event:
            self._events = {}
            return True
        if event not in self._events:
            raise Observable.EventNotFound(event)
        if not func:
            self._events[event] = []
            return True
        if not isinstance(func, list):
            func = [func]
        for f in func:
            if f not in self._events[event]:
                raise Observable.HandlerNotFound(event, f.func_name)
            self._events[event].remove(f)

    def trigger(self, event, *args, **kwargs):
        """trigger all functions from an event"""
        if event not in self._events:
            raise Observable.EventNotFound(event)

        handled = False
        for func in self._events[event]:
            func(*args, **kwargs)
            handled = True
        return handled
<|endoftext|>


<commit_before># !/usr/bin/python
# Copyright 2011 Lockheed Martin

'''
Base couchdb threshold change handler class.

Created on August 18, 2011

@author: jpoyau
'''

import pprint
import logging
from lr.lib import MonitorChanges
import atexit
from lr.model import ResourceDataModel
from distributable_handler import DistributableHandler
from resource_data_handler import ResourceDataHandler
from update_views_handler import  UpdateViewsHandler
from distribute_threshold_handler import DistributeThresholdHandler
from track_last_sequence import TrackLastSequence
from incoming_copy_handler import IncomingCopyHandler
from pylons import config

appConfig = config['app_conf']
log = logging.getLogger(__name__)

_RESOURCE_DATA_CHANGE_ID =  "_local/Last_Processed_Change_Sequence"

_RESOURCE_DATA_CHANGE_HANDLERS=[
    TrackLastSequence(_RESOURCE_DATA_CHANGE_ID),
    UpdateViewsHandler(appConfig['couchdb.threshold.viewupdate']),
    DistributeThresholdHandler(appConfig['couchdb.threshold.distributes'])
    ]

_INCOMING_CHANGE_HANDLERS=[TrackLastSequence(_RESOURCE_DATA_CHANGE_ID),
    IncomingCopyHandler()
    ]

try:
    incomingDB = appConfig['couchdb.db.incoming']
except:
    incomingDB = 'incoming'

def _getLastSavedSequence():
    lastSavedSequence = -1
    if _RESOURCE_DATA_CHANGE_ID in ResourceDataModel._defaultDB:
        lastSavedSequence=ResourceDataModel._defaultDB[_RESOURCE_DATA_CHANGE_ID][TrackLastSequence._LAST_CHANGE_SEQ]
    return lastSavedSequence

def monitorResourceDataChanges():
    options = {'since':_getLastSavedSequence()}
    log.debug("\n\n-----"+pprint.pformat(options)+"------\n\n")

    changeMonitorOptions = {appConfig['couchdb.db.resourcedata']:_RESOURCE_DATA_CHANGE_HANDLERS,
                            incomingDB:_INCOMING_CHANGE_HANDLERS}

    changeMonitors = []
    for db, handler in changeMonitorOptions.items():
        changeMonitor = MonitorChanges(appConfig['couchdb.url'],
                                                            db,
                                                            handler,
                                                            options)
        changeMonitor.start()
        changeMonitors.append(changeMonitor)

    #changeMonitor.start(threading.current_thread())
    def atExitHandler():

        for chMon in changeMonitors:
            chMon.terminate()
        #changeMonitor.terminate()
        log.debug("Last change {0}\n\n".format(changeMonitor._lastChangeSequence))

    atexit.register(atExitHandler)
<commit_msg>switched handlers around<commit_after># !/usr/bin/python
# Copyright 2011 Lockheed Martin

'''
Base couchdb threshold change handler class.

Created on August 18, 2011

@author: jpoyau
'''

import pprint
import logging
from lr.lib import MonitorChanges
import atexit
from lr.model import ResourceDataModel
from distributable_handler import DistributableHandler
from resource_data_handler import ResourceDataHandler
from update_views_handler import  UpdateViewsHandler
from distribute_threshold_handler import DistributeThresholdHandler
from track_last_sequence import TrackLastSequence
from incoming_copy_handler import IncomingCopyHandler
from pylons import config

appConfig = config['app_conf']
log = logging.getLogger(__name__)

_RESOURCE_DATA_CHANGE_ID =  "_local/Last_Processed_Change_Sequence"

_RESOURCE_DATA_CHANGE_HANDLERS=[
    TrackLastSequence(_RESOURCE_DATA_CHANGE_ID),
    UpdateViewsHandler(appConfig['couchdb.threshold.viewupdate']),
    DistributeThresholdHandler(appConfig['couchdb.threshold.distributes'])
    ]

_INCOMING_CHANGE_HANDLERS=[
    TrackLastSequence(_RESOURCE_DATA_CHANGE_ID),
    IncomingCopyHandler()
    ]

try:
    incomingDB = appConfig['couchdb.db.incoming']
except:
    incomingDB = 'incoming'

def _getLastSavedSequence():
    lastSavedSequence = -1
    if _RESOURCE_DATA_CHANGE_ID in ResourceDataModel._defaultDB:
        lastSavedSequence=ResourceDataModel._defaultDB[_RESOURCE_DATA_CHANGE_ID][TrackLastSequence._LAST_CHANGE_SEQ]
    return lastSavedSequence

def monitorResourceDataChanges():
    options = {'since':_getLastSavedSequence()}
    log.debug("\n\n-----"+pprint.pformat(options)+"------\n\n")

    changeMonitorOptions = {appConfig['couchdb.db.resourcedata']:_RESOURCE_DATA_CHANGE_HANDLERS,
                            incomingDB:_INCOMING_CHANGE_HANDLERS}

    changeMonitors = []
    for db, handler in changeMonitorOptions.items():
        changeMonitor = MonitorChanges(appConfig['couchdb.url'],
                                                            db,
                                                            handler,
                                                            options)
        changeMonitor.start()
        changeMonitors.append(changeMonitor)

    #changeMonitor.start(threading.current_thread())
    def atExitHandler():

        for chMon in changeMonitors:
            chMon.terminate()
        #changeMonitor.terminate()
        log.debug("Last change {0}\n\n".format(changeMonitor._lastChangeSequence))

    atexit.register(atExitHandler)
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import calendar
from datetime import timedelta ,datetime
from time import strptime
from glob import glob
import re
import string
import os

from ocw.dataset import Dataset
import ocw.utils as utils

import netCDF4
import numpy
import numpy.ma as ma

LAT_NAMES = ['x', 'rlat', 'rlats', 'lat', 'lats', 'latitude', 'latitudes']
LON_NAMES = ['y', 'rlon', 'rlons', 'lon', 'lons', 'longitude', 'longitudes']
TIME_NAMES = ['time', 'times', 'date', 'dates', 'julian']

def _get_netcdf_variable_name(valid_var_names, netcdf, netcdf_var):
    ''' Determine if one of a set of variable names are in a NetCDF Dataset.

    Looks for an occurrence of a valid_var_name in the NetCDF variable data.
    This is useful for automatically determining the names of the lat, lon,
    and time variable names inside of a dataset object.

    :param valid_var_names: The possible variable names to search for in
        the netCDF object.
    :type valid_var_names: List of Strings
    :param netcdf: The netCDF Dataset object in which to check for
        valid_var_names.
    :type netcdf: netcdf4.Dataset
    :param netcdf_var: The relevant variable name to search over in the
        netcdf object. This is used to narrow down the search for valid
        variable names by first checking the desired variable's dimension
        values for one or more of the valid variable names.

    :returns: The variable from valid_var_names that it locates in
        the netCDF object.

    :raises ValueError: When unable to locate a single matching variable
        name in the NetCDF Dataset from the supplied list of valid variable
        names.
    '''

    # Check for valid variable names in netCDF variable dimensions
    dimensions = netcdf.variables[netcdf_var].dimensions
    dims_lower = [dim.encode().lower() for dim in dimensions]

    intersect = set(valid_var_names).intersection(dims_lower)

    if len(intersect) == 1:
        # Retrieve the name of the dimension where we found the matching
        # variable name
        index = dims_lower.index(intersect.pop())
        dimension_name = dimensions[index].encode()

        # Locate all of the variables that share the dimension that we matched
        # earlier. If the dimension's name matches then that variable is
        # potentially what we want to return to the user.
        possible_vars = []
        for var in netcdf.variables.keys():
            var_dimensions = netcdf.variables[var].dimensions

            # Skip any dimensions are > 1D
            if len(var_dimensions) != 1:
                continue

            if var_dimensions[0].encode() == dimension_name:
                possible_vars.append(var)

        # If there are multiple variables with matching dimension names then we
        # aren't able to determining the correct variable name using the
        # variable dimensions. We need to try a different approach. Otherwise,
        # we're done!
        if len(possible_vars) == 1:
            return possible_vars[0]

    # Check for valid variable names in netCDF variable names
    variables = netcdf.variables.keys()
    vars_lower = [var.encode().lower() for var in variables]

    intersect = set(valid_var_names).intersection(vars_lower)

    if len(intersect) == 1:
        index = vars_lower.index(intersect.pop())
        return variables[index]

    # If we couldn't locate a single matching valid variable then we're unable
    # to automatically determine the variable names for the user.
    error = (
        "Unable to locate a single matching variable name from the "
        "supplied list of valid variable names. "
    )
    raise ValueError(error)

def load_WRF_2d_files(file_path,
                      filename_pattern,
                      variable_name,
                      name=''):
    ''' Load multiple WRF (or nuWRF) original output files containing 2D fields such as precipitation and surface variables into a Dataset.
    The dataset can be spatially subset.
    :param file_path: Directory to the NetCDF file to load.
    :type file_path: :mod:`string`
    :param filename_pattern: Path to the NetCDF file to load.
    :type filename_pattern: :list:`string`
    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`
    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`
    :returns: An OCW Dataset object with the requested variable's data from
        the NetCDF file.
    :rtype: :class:`dataset.Dataset`
    :raises ValueError:
    '''

    WRF_files = []
    for pattern in filename_pattern:
        WRF_files.extend(glob(file_path + pattern))
    WRF_files.sort()

    file_object_first = netCDF4.Dataset(WRF_files[0])
    lats = file_object_first.variables['XLAT'][0,:]
    lons = file_object_first.variables['XLONG'][0,:]

    times = []
    for ifile, file in enumerate(WRF_files):
        file_object = netCDF4.Dataset(file)
        time_struct_parsed = strptime(file[-19:],"%Y-%m-%d_%H:%M:%S")
        for ihour in numpy.arange(24):
            times.append(datetime(*time_struct_parsed[:6]) + timedelta(hours=ihour))
        values0= file_object.variables[variable_name][:]
        if ifile == 0:
            values = file_object.variables[variable_name][:]
        else:
            values = numpy.concatenate((values, file_object.variables[variable_name][:]))
        file_object.close()
    times = numpy.array(times)
    return Dataset(lats, lons, times, values, variable_name, name=name)

def load_file(file_path,
              variable_name,
              variable_unit = None,
              elevation_index=0,
              name='',
              lat_name=None,
              lon_name=None,
              time_name=None):
    ''' Load a NetCDF file into a Dataset.

    :param file_path: Path to the NetCDF file to load.
    :type file_path: :mod:`string`

    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`

    :param variable_unit: (Optional) The variable unit to load from the NetCDF file.
    :type variable_unit: :mod:`string`

    :param elevation_index: (Optional) The elevation index for which data should
        be returned. Climate data is often times 4 dimensional data. Some
        datasets will have readins at different height/elevation levels. OCW
        expects 3D data so a single layer needs to be stripped out when loading.
        By default, the first elevation layer is used. If desired you may
        specify the elevation value to use.
    :type elevation_index: :class:`int`

    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`

    :param lat_name: (Optional) The latitude variable name to extract from the
        dataset.
    :type lat_name: :mod:`string`

    :param lon_name: (Optional) The longitude variable name to extract from the
        dataset.
    :type lon_name: :mod:`string`

    :param time_name: (Optional) The time variable name to extract from the
        dataset.
    :type time_name: :mod:`string`

    :returns: An OCW Dataset object with the requested variable's data from
        the NetCDF file.
    :rtype: :class:`dataset.Dataset`

    :raises ValueError: When the specified file path cannot be loaded by ndfCDF4
        or when the lat/lon/time variable name cannot be determined
        automatically.
    '''

    try:
        netcdf = netCDF4.Dataset(file_path, mode='r')
    except RuntimeError:
        err = "Dataset filepath is invalid. Please ensure it is correct."
        raise ValueError(err)
    except:
        err = (
            "The given file cannot be loaded. Please ensure that it is a valid "
            "NetCDF file. If problems persist, report them to the project's "
            "mailing list."
        )
        raise ValueError(err)

    if not lat_name:
        lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
    if not lon_name:
        lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
    if not time_name:
        time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)

    lats = netcdf.variables[lat_name][:]
    lons = netcdf.variables[lon_name][:]
    time_raw_values = netcdf.variables[time_name][:]
    times = utils.decode_time_values(netcdf, time_name)
    times = numpy.array(times)
    values = ma.array(netcdf.variables[variable_name][:])
    variable_unit = netcdf.variables[variable_name].units

    # If the values are 4D then we need to strip out the elevation index
    if len(values.shape) == 4:
        # Determine the set of possible elevation dimension names excluding
        # the list of names that are used for the lat, lon, and time values.
        dims = netcdf.variables[variable_name].dimensions
        dimension_names = [dim_name.encode() for dim_name in dims]
        lat_lon_time_var_names = [lat_name, lon_name, time_name]

        elev_names = set(dimension_names) - set(lat_lon_time_var_names)

        # Grab the index value for the elevation values
        level_index = dimension_names.index(elev_names.pop())

        # Strip out the elevation values so we're left with a 3D array.
        if level_index == 0:
            values = values [elevation_index,:,:,:]
        elif level_index == 1:
            values = values [:,elevation_index,:,:]
        elif level_index == 2:
            values = values [:,:,elevation_index,:]
        else:
            values = values [:,:,:,elevation_index]

    origin = {
        'source': 'local',
        'path': file_path,
        'lat_name': lat_name,
        'lon_name': lon_name,
        'time_name': time_name
    }
    if elevation_index != 0: origin['elevation_index'] = elevation_index

    return Dataset(lats, lons, times, values, variable=variable_name,
                   units=variable_unit, name=name, origin=origin)

def load_multiple_files(file_path,
                        variable_name,
                        dataset_name='model',
                        variable_unit=None,
                        lat_name=None,
                        lon_name=None,
                        time_name=None):
    ''' load multiple netcdf files with common filename pattern and return an array of OCW datasets

    :param file_path: directory name and common file name patterns where the NetCDF files to load are stored.
    :type file_path: :mod:`string`
    :param dataset_name: a name of dataset when reading a single file
    :type dataset_name: :mod:'string'
    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`
    :param variable_unit: (Optional) The variable unit to load from the NetCDF file.
    :type variable_unit: :mod:`string`
    :param elevation_index: (Optional) The elevation index for which data should
        be returned. Climate data is often times 4 dimensional data. Some
        datasets will have readins at different height/elevation levels. OCW
        expects 3D data so a single layer needs to be stripped out when loading.
        By default, the first elevation layer is used. If desired you may
        specify the elevation value to use.
    :param lat_name: (Optional) The latitude variable name to extract from the
        dataset.
    :type lat_name: :mod:`string`
    :param lon_name: (Optional) The longitude variable name to extract from the
        dataset.
    :type lon_name: :mod:`string`
    :param time_name: (Optional) The time variable name to extract from the
        dataset.
    :type time_name: :mod:`string`
    :returns: An array of OCW Dataset objects, an array of dataset names
    :rtype: :class:`list`
    '''

    data_filenames = []
    data_filenames.extend(glob(file_path))
    data_filenames.sort()

    # number of files
    ndata = len(data_filenames)
    if ndata == 1:
        data_name = [dataset_name]
    else:
        data_name = []
        data_filenames_reversed = []
        for element in data_filenames:
            data_filenames_reversed.append(element[::-1])
        prefix = os.path.commonprefix(data_filenames)
        postfix = os.path.commonprefix(data_filenames_reversed)[::-1]
        for element in data_filenames:
            data_name.append(element.replace(prefix,'').replace(postfix,''))

    datasets = []
    for ifile,filename in enumerate(data_filenames):
        datasets.append(load_file(filename, variable_name, variable_unit, name=data_name[ifile],
                        lat_name=lat_name, lon_name=lon_name, time_name=time_name))

    return datasets, data_name

def load_GPM_IMERG_files(file_path=None,
                      filename_pattern=None,
                      filelist=None,
                      variable_name='precipitationCal',
                      name='GPM_IMERG'):
    ''' Load multiple GPM Level 3 IMEGE files containing calibrated precipitation and generate an OCW Dataset obejct.
    :param file_path: Directory to the HDF files to load.
    :type file_path: :mod:`string`
    :param filename_pattern: Path to the HDF files to load.
    :type filename_pattern: :list:`string`
    :param filelist: A list of filenames
    :type filelist: :list:`string`
    :param variable_name: The variable name to load from the HDF file.
    :type variable_name: :mod:`string`
    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`
    :returns: An OCW Dataset object with the requested variable's data from
        the HDF file.
    :rtype: :class:`dataset.Dataset`
    :raises ValueError:
    '''

    if not filelist:
        GPM_files = []
        for pattern in filename_pattern:
            GPM_files.extend(glob(file_path + pattern))
    else:
        GPM_files = [line.rstrip('\n') for line in open(filelist)]

    GPM_files.sort()

    file_object_first = h5py.File(GPM_files[0])
    lats = file_object_first['Grid']['lat'][:]
    lons = file_object_first['Grid']['lon'][:]

    lons, lats = numpy.meshgrid(lons, lats)

    variable_unit = "mm/hr"

    times = []
    nfile = len(GPM_files)
    for ifile, file in enumerate(GPM_files):
        print 'Reading file '+str(ifile+1)+'/'+str(nfile), file
        file_object = h5py.File(file)
        time_struct_parsed = strptime(file[-39:-23],"%Y%m%d-S%H%M%S")
        times.append(datetime(*time_struct_parsed[:6]))
        values0= numpy.transpose(ma.masked_less(file_object['Grid'][variable_name][:], 0.))
        values0= numpy.expand_dims(values0, axis=0)
        if ifile == 0:
            values = values0
        else:
            values = numpy.concatenate((values, values0))
        file_object.close()
    times = numpy.array(times)
    return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)
<commit_msg>h5py library is added<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import calendar
from datetime import timedelta ,datetime
from time import strptime
from glob import glob
import re
import string
import os

from ocw.dataset import Dataset
import ocw.utils as utils

import netCDF4
import h5py
import numpy
import numpy.ma as ma

LAT_NAMES = ['x', 'rlat', 'rlats', 'lat', 'lats', 'latitude', 'latitudes']
LON_NAMES = ['y', 'rlon', 'rlons', 'lon', 'lons', 'longitude', 'longitudes']
TIME_NAMES = ['time', 'times', 'date', 'dates', 'julian']

def _get_netcdf_variable_name(valid_var_names, netcdf, netcdf_var):
    ''' Determine if one of a set of variable names are in a NetCDF Dataset.

    Looks for an occurrence of a valid_var_name in the NetCDF variable data.
    This is useful for automatically determining the names of the lat, lon,
    and time variable names inside of a dataset object.

    :param valid_var_names: The possible variable names to search for in
        the netCDF object.
    :type valid_var_names: List of Strings
    :param netcdf: The netCDF Dataset object in which to check for
        valid_var_names.
    :type netcdf: netcdf4.Dataset
    :param netcdf_var: The relevant variable name to search over in the
        netcdf object. This is used to narrow down the search for valid
        variable names by first checking the desired variable's dimension
        values for one or more of the valid variable names.

    :returns: The variable from valid_var_names that it locates in
        the netCDF object.

    :raises ValueError: When unable to locate a single matching variable
        name in the NetCDF Dataset from the supplied list of valid variable
        names.
    '''

    # Check for valid variable names in netCDF variable dimensions
    dimensions = netcdf.variables[netcdf_var].dimensions
    dims_lower = [dim.encode().lower() for dim in dimensions]

    intersect = set(valid_var_names).intersection(dims_lower)

    if len(intersect) == 1:
        # Retrieve the name of the dimension where we found the matching
        # variable name
        index = dims_lower.index(intersect.pop())
        dimension_name = dimensions[index].encode()

        # Locate all of the variables that share the dimension that we matched
        # earlier. If the dimension's name matches then that variable is
        # potentially what we want to return to the user.
        possible_vars = []
        for var in netcdf.variables.keys():
            var_dimensions = netcdf.variables[var].dimensions

            # Skip any dimensions are > 1D
            if len(var_dimensions) != 1:
                continue

            if var_dimensions[0].encode() == dimension_name:
                possible_vars.append(var)

        # If there are multiple variables with matching dimension names then we
        # aren't able to determining the correct variable name using the
        # variable dimensions. We need to try a different approach. Otherwise,
        # we're done!
        if len(possible_vars) == 1:
            return possible_vars[0]

    # Check for valid variable names in netCDF variable names
    variables = netcdf.variables.keys()
    vars_lower = [var.encode().lower() for var in variables]

    intersect = set(valid_var_names).intersection(vars_lower)

    if len(intersect) == 1:
        index = vars_lower.index(intersect.pop())
        return variables[index]

    # If we couldn't locate a single matching valid variable then we're unable
    # to automatically determine the variable names for the user.
    error = (
        "Unable to locate a single matching variable name from the "
        "supplied list of valid variable names. "
    )
    raise ValueError(error)

def load_WRF_2d_files(file_path,
                      filename_pattern,
                      variable_name,
                      name=''):
    ''' Load multiple WRF (or nuWRF) original output files containing 2D fields such as precipitation and surface variables into a Dataset.
    The dataset can be spatially subset.
    :param file_path: Directory to the NetCDF file to load.
    :type file_path: :mod:`string`
    :param filename_pattern: Path to the NetCDF file to load.
    :type filename_pattern: :list:`string`
    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`
    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`
    :returns: An OCW Dataset object with the requested variable's data from
        the NetCDF file.
    :rtype: :class:`dataset.Dataset`
    :raises ValueError:
    '''

    WRF_files = []
    for pattern in filename_pattern:
        WRF_files.extend(glob(file_path + pattern))
    WRF_files.sort()

    file_object_first = netCDF4.Dataset(WRF_files[0])
    lats = file_object_first.variables['XLAT'][0,:]
    lons = file_object_first.variables['XLONG'][0,:]

    times = []
    for ifile, file in enumerate(WRF_files):
        file_object = netCDF4.Dataset(file)
        time_struct_parsed = strptime(file[-19:],"%Y-%m-%d_%H:%M:%S")
        for ihour in numpy.arange(24):
            times.append(datetime(*time_struct_parsed[:6]) + timedelta(hours=ihour))
        values0= file_object.variables[variable_name][:]
        if ifile == 0:
            values = file_object.variables[variable_name][:]
        else:
            values = numpy.concatenate((values, file_object.variables[variable_name][:]))
        file_object.close()
    times = numpy.array(times)
    return Dataset(lats, lons, times, values, variable_name, name=name)

def load_file(file_path,
              variable_name,
              variable_unit = None,
              elevation_index=0,
              name='',
              lat_name=None,
              lon_name=None,
              time_name=None):
    ''' Load a NetCDF file into a Dataset.

    :param file_path: Path to the NetCDF file to load.
    :type file_path: :mod:`string`

    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`

    :param variable_unit: (Optional) The variable unit to load from the NetCDF file.
    :type variable_unit: :mod:`string`

    :param elevation_index: (Optional) The elevation index for which data should
        be returned. Climate data is often times 4 dimensional data. Some
        datasets will have readins at different height/elevation levels. OCW
        expects 3D data so a single layer needs to be stripped out when loading.
        By default, the first elevation layer is used. If desired you may
        specify the elevation value to use.
    :type elevation_index: :class:`int`

    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`

    :param lat_name: (Optional) The latitude variable name to extract from the
        dataset.
    :type lat_name: :mod:`string`

    :param lon_name: (Optional) The longitude variable name to extract from the
        dataset.
    :type lon_name: :mod:`string`

    :param time_name: (Optional) The time variable name to extract from the
        dataset.
    :type time_name: :mod:`string`

    :returns: An OCW Dataset object with the requested variable's data from
        the NetCDF file.
    :rtype: :class:`dataset.Dataset`

    :raises ValueError: When the specified file path cannot be loaded by ndfCDF4
        or when the lat/lon/time variable name cannot be determined
        automatically.
    '''

    try:
        netcdf = netCDF4.Dataset(file_path, mode='r')
    except RuntimeError:
        err = "Dataset filepath is invalid. Please ensure it is correct."
        raise ValueError(err)
    except:
        err = (
            "The given file cannot be loaded. Please ensure that it is a valid "
            "NetCDF file. If problems persist, report them to the project's "
            "mailing list."
        )
        raise ValueError(err)

    if not lat_name:
        lat_name = _get_netcdf_variable_name(LAT_NAMES, netcdf, variable_name)
    if not lon_name:
        lon_name = _get_netcdf_variable_name(LON_NAMES, netcdf, variable_name)
    if not time_name:
        time_name = _get_netcdf_variable_name(TIME_NAMES, netcdf, variable_name)

    lats = netcdf.variables[lat_name][:]
    lons = netcdf.variables[lon_name][:]
    time_raw_values = netcdf.variables[time_name][:]
    times = utils.decode_time_values(netcdf, time_name)
    times = numpy.array(times)
    values = ma.array(netcdf.variables[variable_name][:])
    variable_unit = netcdf.variables[variable_name].units

    # If the values are 4D then we need to strip out the elevation index
    if len(values.shape) == 4:
        # Determine the set of possible elevation dimension names excluding
        # the list of names that are used for the lat, lon, and time values.
        dims = netcdf.variables[variable_name].dimensions
        dimension_names = [dim_name.encode() for dim_name in dims]
        lat_lon_time_var_names = [lat_name, lon_name, time_name]

        elev_names = set(dimension_names) - set(lat_lon_time_var_names)

        # Grab the index value for the elevation values
        level_index = dimension_names.index(elev_names.pop())

        # Strip out the elevation values so we're left with a 3D array.
        if level_index == 0:
            values = values [elevation_index,:,:,:]
        elif level_index == 1:
            values = values [:,elevation_index,:,:]
        elif level_index == 2:
            values = values [:,:,elevation_index,:]
        else:
            values = values [:,:,:,elevation_index]

    origin = {
        'source': 'local',
        'path': file_path,
        'lat_name': lat_name,
        'lon_name': lon_name,
        'time_name': time_name
    }
    if elevation_index != 0: origin['elevation_index'] = elevation_index

    return Dataset(lats, lons, times, values, variable=variable_name,
                   units=variable_unit, name=name, origin=origin)

def load_multiple_files(file_path,
                        variable_name,
                        dataset_name='model',
                        variable_unit=None,
                        lat_name=None,
                        lon_name=None,
                        time_name=None):
    ''' load multiple netcdf files with common filename pattern and return an array of OCW datasets

    :param file_path: directory name and common file name patterns where the NetCDF files to load are stored.
    :type file_path: :mod:`string`
    :param dataset_name: a name of dataset when reading a single file
    :type dataset_name: :mod:'string'
    :param variable_name: The variable name to load from the NetCDF file.
    :type variable_name: :mod:`string`
    :param variable_unit: (Optional) The variable unit to load from the NetCDF file.
    :type variable_unit: :mod:`string`
    :param elevation_index: (Optional) The elevation index for which data should
        be returned. Climate data is often times 4 dimensional data. Some
        datasets will have readins at different height/elevation levels. OCW
        expects 3D data so a single layer needs to be stripped out when loading.
        By default, the first elevation layer is used. If desired you may
        specify the elevation value to use.
    :param lat_name: (Optional) The latitude variable name to extract from the
        dataset.
    :type lat_name: :mod:`string`
    :param lon_name: (Optional) The longitude variable name to extract from the
        dataset.
    :type lon_name: :mod:`string`
    :param time_name: (Optional) The time variable name to extract from the
        dataset.
    :type time_name: :mod:`string`
    :returns: An array of OCW Dataset objects, an array of dataset names
    :rtype: :class:`list`
    '''

    data_filenames = []
    data_filenames.extend(glob(file_path))
    data_filenames.sort()

    # number of files
    ndata = len(data_filenames)
    if ndata == 1:
        data_name = [dataset_name]
    else:
        data_name = []
        data_filenames_reversed = []
        for element in data_filenames:
            data_filenames_reversed.append(element[::-1])
        prefix = os.path.commonprefix(data_filenames)
        postfix = os.path.commonprefix(data_filenames_reversed)[::-1]
        for element in data_filenames:
            data_name.append(element.replace(prefix,'').replace(postfix,''))

    datasets = []
    for ifile,filename in enumerate(data_filenames):
        datasets.append(load_file(filename, variable_name, variable_unit, name=data_name[ifile],
                        lat_name=lat_name, lon_name=lon_name, time_name=time_name))

    return datasets, data_name

def load_GPM_IMERG_files(file_path=None,
                      filename_pattern=None,
                      filelist=None,
                      variable_name='precipitationCal',
                      name='GPM_IMERG'):
    ''' Load multiple GPM Level 3 IMEGE files containing calibrated precipitation and generate an OCW Dataset obejct.
    :param file_path: Directory to the HDF files to load.
    :type file_path: :mod:`string`
    :param filename_pattern: Path to the HDF files to load.
    :type filename_pattern: :list:`string`
    :param filelist: A list of filenames
    :type filelist: :list:`string`
    :param variable_name: The variable name to load from the HDF file.
    :type variable_name: :mod:`string`
    :param name: (Optional) A name for the loaded dataset.
    :type name: :mod:`string`
    :returns: An OCW Dataset object with the requested variable's data from
        the HDF file.
    :rtype: :class:`dataset.Dataset`
    :raises ValueError:
    '''

    if not filelist:
        GPM_files = []
        for pattern in filename_pattern:
            GPM_files.extend(glob(file_path + pattern))
    else:
        GPM_files = [line.rstrip('\n') for line in open(filelist)]

    GPM_files.sort()

    file_object_first = h5py.File(GPM_files[0])
    lats = file_object_first['Grid']['lat'][:]
    lons = file_object_first['Grid']['lon'][:]

    lons, lats = numpy.meshgrid(lons, lats)

    variable_unit = "mm/hr"

    times = []
    nfile = len(GPM_files)
    for ifile, file in enumerate(GPM_files):
        print 'Reading file '+str(ifile+1)+'/'+str(nfile), file
        file_object = h5py.File(file)
        time_struct_parsed = strptime(file[-39:-23],"%Y%m%d-S%H%M%S")
        times.append(datetime(*time_struct_parsed[:6]))
        values0= numpy.transpose(ma.masked_less(file_object['Grid'][variable_name][:], 0.))
        values0= numpy.expand_dims(values0, axis=0)
        if ifile == 0:
            values = values0
        else:
            values = numpy.concatenate((values, values0))
        file_object.close()
    times = numpy.array(times)
    return Dataset(lats, lons, times, values, variable_name, units=variable_unit, name=name)
<|endoftext|>


<commit_before>
from Wikidata import Wikidata
from WikidataTitleResolver import WikidataTitleResolver

class ISO31661Resolver(WikidataTitleResolver):
    def get_flag(self, code):
        code = 'ISO 3166-2:{0}'.format(code)

        # Call base class get_flag method
        return super(ISO31661Resolver, self).get_flag(code)
<commit_msg>ISO 3166-1 codes are uppercase.<commit_after>
from Wikidata import Wikidata
from WikidataTitleResolver import WikidataTitleResolver

class ISO31661Resolver(WikidataTitleResolver):
    def get_flag(self, code):
        code = 'ISO 3166-2:{0}'.format(code.upper())

        # Call base class get_flag method
        return super(ISO31661Resolver, self).get_flag(code)
<|endoftext|>