<commit_before># -*- coding: utf-8 -*-

'''https://www.bitstamp.net/api/'''

import requests
from decimal import Decimal
import time
from cryptotik.common import (headers, ExchangeWrapper,
                              NormalizedExchangeWrapper)
from cryptotik.exceptions import (InvalidBaseCurrencyError,
                                  InvalidDelimiterError,
                                  APIError)
import hmac
import hashlib
from datetime import datetime

class Bitstamp(ExchangeWrapper):

    def __init__(self, apikey=None, secret=None, customer_id=None,
                 timeout=None, proxy=None):

        if apikey:
            self._apikey = apikey
            self._secret = secret
            self._customer_id = customer_id

        if proxy:
            assert proxy.startswith('https'), {'Error': 'Only https proxies supported.'}
        self.proxy = {'https': proxy}

        if not timeout:
            self.timeout = (8, 15)
        else:
            self.timeout = timeout

        self.api_session = requests.Session()

    public_commands = ("ticker", "transactions", "order_book")
    private_commands = ("balance", "user_transactions", "open_orders", "order_status",
                        "cancel_order", "cancel_all_orders", "buy",
                        "sell")

    name = 'bitstamp'
    url = 'https://www.bitstamp.net/'
    api_url = url + 'api/'
    delimiter = ""
    case = "lower"
    headers = headers
    _markets = 'btcusd, btceur, eurusd, xrpusd, xrpeur, xrpbtc, ltcusd, ltceur, ltcbtc, ethusd, etheur, ethbtc'.split(', ')
    maker_fee, taker_fee = 0.002, 0.002
    quote_order = 0
    base_currencies = ['usd', 'eur', 'btc']

    def get_nonce(self):
        '''return nonce integer'''

        nonce = getattr(self, '_nonce', 0)
        if nonce:
            nonce += 1
        # If the unix time is greater though, use that instead (helps low
        # concurrency multi-threaded apps always call with the largest nonce).
        self._nonce = max(int(time.time() * 1000000), nonce)
        return self._nonce

    def get_base_currencies(self):
        raise NotImplementedError

    @classmethod
    def format_pair(cls, pair):
        """format the pair argument to format understood by remote API."""

        pair = pair.replace("-", cls.delimiter)

        if not pair.islower():
            return pair.lower()
        else:
            return pair

    def _verify_response(self, response):
        '''verify if API responded properly and raise apropriate error.'''

        if 'v2' in response.url:  # works only for v2 calls
            try:
                if response.json()['error']:
                    raise APIError(response.json()['reason'])
            except (KeyError, TypeError):
                pass

    def _generate_signature(self, message):

        return hmac.new(self._secret.encode('utf-8'),
                        msg=message.encode('utf-8'),
                        digestmod=hashlib.sha256).hexdigest().upper()

    def api(self, command):
        """call remote API"""

        try:
            response = self.api_session.get(self.api_url + 'v2/' + command, headers=self.headers,
                                            timeout=self.timeout, proxies=self.proxy)

            response.raise_for_status()

        except requests.exceptions.HTTPError as e:
            raise APIError(e)

        self._verify_response(response)
        return response.json()

    def private_api(self, command, data={}):
        '''handles private api methods'''

        if not self._customer_id or not self._apikey or not self._secret:
            raise ValueError("A Key, Secret and customer_id required!")

        nonce = self.get_nonce()
        data['key'] = self._apikey
        message = str(nonce) + self._customer_id + self._apikey
        data['signature'] = self._generate_signature(message)
        data['nonce'] = nonce

        try:
            response = self.api_session.post(url=self.api_url + command,
                                             data=data,
                                             headers=self.headers,
                                             timeout=self.timeout,
                                             proxies=self.proxy)

            response.raise_for_status()

        except requests.exceptions.HTTPError as e:
            print(e)

        self._verify_response(response)
        return response.json()

    def get_markets(self):
        '''get all market pairs supported by the exchange'''

        return self._markets

    def get_summaries(self):
        '''get summary of all active markets'''

        raise NotImplementedError

    def get_market_ticker(self, pair):
        """return ticker for market <pair>"""

        pair = self.format_pair(pair)
        return self.api("ticker/" + pair)

    def get_market_orders(self, pair):
        """returns market order book for <pair>"""

        pair = self.format_pair(pair)
        return self.api("order_book/" + pair)

    def get_market_trade_history(self, pair, since="hour"):
        """get market trade history; since {minute, hour, day}"""

        pair = self.format_pair(pair)

        return self.api("transactions/" + pair + "/?time={0}".format(since))

    def get_market_volume(self, pair):
        '''return market volume [of last 24h]'''

        pair = self.format_pair(pair)

        return self.get_market_ticker(pair)['volume']

    ########################
    # Private methods bellow
    ########################

    def get_balances(self, coin=None):
        '''Returns the values relevant to the specified <coin> parameter.'''

        if not coin:
            return self.private_api("v2/balance/")
        else:
            return {k: v for k, v in self.private_api("v2/balance/").items() if k.startswith(coin.lower())}

    def get_deposit_address(self, coin=None):
        '''get deposit address'''

        if coin == 'btc':
            command = 'bitcoin_deposit_address/'
        if coin == 'ltc':
            command = 'v2/ltc_address/'
        if coin == 'eth':
            command = 'v2/eth_address/'
        if coin == 'xrp':
            command = 'ripple_address/'
        if coin == 'bch':
            command = 'v2/bch_address/'

        return self.private_api(command)

    def get_liquidation_address(self, fiat):
        '''Creates new liquidation address which will automatically sell your BTC for specified liquidation_currency.'''

        return self.private_api('v2/liquidation_address/new/',
                                data={'liquidation_currency': fiat.lower()})

    def get_liquidation_address_info(self, address=None):
        '''Shows transactions (BTC to liquidation_currency) for liquidation address.'''

        return self.private_api('v2/liquidation_address/info/',
                                data={'address': address})

    def buy_limit(self, pair, rate, amount, daily_order=False):
        '''submit limit buy order

        daily_order: opens buy limit order which will be canceled at 0:00 UTC unless it already has been executed. Possible value: True'''

        pair = self.format_pair(pair)
        return self.private_api('v2/buy/{}/'.format(pair),
                                data={'amount': amount,
                                      'price': rate,
                                      'daily_order': daily_order
                                      }
                                )

    def buy_market(self, pair, amount):
        '''submit market buy order'''

        pair = self.format_pair(pair)
        return self.private_api('v2/buy/market/{}/'.format(pair),
                                data={'amount': amount}
                                )

    def sell_limit(self, pair, rate, amount, daily_order=False):
        '''submit limit sell order'''

        pair = self.format_pair(pair)
        return self.private_api('v2/sell/{}/'.format(pair),
                                data={'amount': amount,
                                      'price': rate,
                                      'daily_order': daily_order
                                      }
                                )

    def sell_market(self, pair, amount):
        '''submit market sell order'''

        pair = self.format_pair(pair)
        return self.private_api('v2/sell/market/{}/'.format(pair),
                                data={'amount': amount})

    def cancel_order(self, order_id):
        '''cancel order by <order_id>'''

        return self.private_api('v2/cancel_order/', data={'id': order_id})

    def cancel_all_orders(self):
        '''cancel all active orders'''

        return self.private_api('cancel_all_orders/')

    def get_open_orders(self, pair):
        '''Get open orders.'''

        pair = self.format_pair(pair)
        return self.private_api("v2/open_orders/{}/".format(pair))

    def get_order(self, order_id):
        '''get order information'''

        return self.private_api('orders_status/', data={'id': order_id})

    def withdraw(self, coin, amount, address):
        '''withdraw cryptocurrency'''

        if coin == 'btc':
            command = 'bitcoin_withdrawal/'
        if coin == 'ltc':
            command = 'v2/ltc_withdrawal/'
        if coin == 'eth':
            command = 'v2/eth_withdrawal/'
        if coin == 'xrp':
            command = 'ripple_withdrawal/'
        if coin == 'bch':
            command = 'v2/bch_withdrawal/'

        return self.private_api(command, data={'amount': amount,
                                               'address': address}
                                )

    def get_transaction_history(self):
        '''Returns the history of transactions.'''

        raise NotImplementedError

    def get_deposit_history(self, coin=None):
        '''get deposit history'''

        raise NotImplementedError

    def get_withdraw_history(self, coin=None, timedelta=50000000):
        '''get withdrawals history'''

        return self.private_api('v2/withdrawal-requests/',
                                data={'timedelta': timedelta})

class BitstampNormalized(Bitstamp, NormalizedExchangeWrapper):

    def __init__(self, apikey=None, secret=None, customer_id=None, timeout=None, proxy=None):
        super(BitstampNormalized, self).__init__(apikey, secret, customer_id, timeout, proxy)

    @classmethod
    def format_pair(self, market_pair):
        """
        Expected input is quote - base.
        Normalize the pair inputs and
        format the pair argument to a format understood by the remote API."""

        market_pair = market_pair.lower()  # bittrex takes lowercase

        if "-" not in market_pair:
            raise InvalidDelimiterError('Agreed upon delimiter is "-".')

        quote, base = market_pair.split('-')

        if base not in self.base_currencies:
            raise InvalidBaseCurrencyError('''Expected input is quote-base, you have provided with {pair}'''.format(pair=market_pair))

        return quote + self.delimiter + base  # for bistamp quote comes first

    @staticmethod
    def _tstamp_to_datetime(timestamp):
        '''convert unix timestamp to datetime'''

        return datetime.fromtimestamp(timestamp)

    @staticmethod
    def _is_sale(s):

        if s == 0:
            return True
        else:
            return False

    def get_market_ticker(self, market):
        '''
        :return :
            dict['ask': float, 'bid': float, 'last': float]
            example: {'ask': float, 'bid': float, 'last': float}
        '''

        ticker = super().get_market_ticker(market)

        return {
            'ask': ticker['ask'],
            'bid': ticker['bid'],
            'last': ticker['last']
        }

    def get_market_trade_history(self, market):
        '''
        :return:
            list -> dict['timestamp': datetime.datetime,
                        'is_sale': bool,
                        'rate': float,
                        'amount': float,
                        'trade_id': any]
        '''

        upstream = super().get_market_trade_history(market)
        downstream = []

        for data in upstream:

            downstream.append({
                'timestamp': self._tstamp_to_datetime(int(data['date'])),
                'is_sale': self._is_sale(data['type']),
                'rate': data['price'],
                'amount': data['amount'],
                'trade_id': data['tid']
            })

        return downstream

    def get_market_orders(self, market):
        '''
        :return:
            dict['bids': list[price, quantity],
                 'asks': list[price, quantity]
                ]
        bids[0] should be first next to the spread
        asks[0] should be first next to the spread
        '''

        upstream = super().get_market_orders(market)

        return {
            'bids': [[i[0], i[1]] for i in upstream['bids']],
            'asks': [[i[0], i[1]] for i in upstream['asks']]
        }

    def get_market_spread(self, market):
        '''return first buy order and first sell order'''

        order_book = super().get_market_orders(market)

        ask = order_book['asks'][0][0]
        bid = order_book['bids'][0][0]

        return Decimal(ask) - Decimal(bid)

    def get_market_depth(self, market):
        '''return sum of all bids and asks'''

        order_book = self.get_market_orders(market)

        return {"bids": sum([Decimal(i[0]) * Decimal(i[1]) for i in order_book["bids"]]),
                "asks": sum([Decimal(i[1]) for i in order_book["asks"]])
                }
<commit_msg>Bitstamp: implement get_market_sell_orders and get_market_buy_orders<commit_after># -*- coding: utf-8 -*-

'''https://www.bitstamp.net/api/'''

import requests
from decimal import Decimal
import time
from cryptotik.common import (headers, ExchangeWrapper,
                              NormalizedExchangeWrapper)
from cryptotik.exceptions import (InvalidBaseCurrencyError,
                                  InvalidDelimiterError,
                                  APIError)
import hmac
import hashlib
from datetime import datetime

class Bitstamp(ExchangeWrapper):

    def __init__(self, apikey=None, secret=None, customer_id=None,
                 timeout=None, proxy=None):

        if apikey:
            self._apikey = apikey
            self._secret = secret
            self._customer_id = customer_id

        if proxy:
            assert proxy.startswith('https'), {'Error': 'Only https proxies supported.'}
        self.proxy = {'https': proxy}

        if not timeout:
            self.timeout = (8, 15)
        else:
            self.timeout = timeout

        self.api_session = requests.Session()

    public_commands = ("ticker", "transactions", "order_book")
    private_commands = ("balance", "user_transactions", "open_orders", "order_status",
                        "cancel_order", "cancel_all_orders", "buy",
                        "sell")

    name = 'bitstamp'
    url = 'https://www.bitstamp.net/'
    api_url = url + 'api/'
    delimiter = ""
    case = "lower"
    headers = headers
    _markets = 'btcusd, btceur, eurusd, xrpusd, xrpeur, xrpbtc, ltcusd, ltceur, ltcbtc, ethusd, etheur, ethbtc'.split(', ')
    maker_fee, taker_fee = 0.002, 0.002
    quote_order = 0
    base_currencies = ['usd', 'eur', 'btc']

    def get_nonce(self):
        '''return nonce integer'''

        nonce = getattr(self, '_nonce', 0)
        if nonce:
            nonce += 1
        # If the unix time is greater though, use that instead (helps low
        # concurrency multi-threaded apps always call with the largest nonce).
        self._nonce = max(int(time.time() * 1000000), nonce)
        return self._nonce

    def get_base_currencies(self):
        raise NotImplementedError

    @classmethod
    def format_pair(cls, pair):
        """format the pair argument to format understood by remote API."""

        pair = pair.replace("-", cls.delimiter)

        if not pair.islower():
            return pair.lower()
        else:
            return pair

    def _verify_response(self, response):
        '''verify if API responded properly and raise apropriate error.'''

        if 'v2' in response.url:  # works only for v2 calls
            try:
                if response.json()['error']:
                    raise APIError(response.json()['reason'])
            except (KeyError, TypeError):
                pass

    def _generate_signature(self, message):

        return hmac.new(self._secret.encode('utf-8'),
                        msg=message.encode('utf-8'),
                        digestmod=hashlib.sha256).hexdigest().upper()

    def api(self, command):
        """call remote API"""

        try:
            response = self.api_session.get(self.api_url + 'v2/' + command, headers=self.headers,
                                            timeout=self.timeout, proxies=self.proxy)

            response.raise_for_status()

        except requests.exceptions.HTTPError as e:
            raise APIError(e)

        self._verify_response(response)
        return response.json()

    def private_api(self, command, data={}):
        '''handles private api methods'''

        if not self._customer_id or not self._apikey or not self._secret:
            raise ValueError("A Key, Secret and customer_id required!")

        nonce = self.get_nonce()
        data['key'] = self._apikey
        message = str(nonce) + self._customer_id + self._apikey
        data['signature'] = self._generate_signature(message)
        data['nonce'] = nonce

        try:
            response = self.api_session.post(url=self.api_url + command,
                                             data=data,
                                             headers=self.headers,
                                             timeout=self.timeout,
                                             proxies=self.proxy)

            response.raise_for_status()

        except requests.exceptions.HTTPError as e:
            print(e)

        self._verify_response(response)
        return response.json()

    def get_markets(self):
        '''get all market pairs supported by the exchange'''

        return self._markets

    def get_summaries(self):
        '''get summary of all active markets'''

        raise NotImplementedError

    def get_market_ticker(self, pair):
        """return ticker for market <pair>"""

        pair = self.format_pair(pair)
        return self.api("ticker/" + pair)

    def get_market_orders(self, pair):
        """returns market order book for <pair>"""

        pair = self.format_pair(pair)
        return self.api("order_book/" + pair)

    def get_market_sell_orders(self, pair):

        return self.get_market_orders(pair)['asks']

    def get_market_buy_orders(self, pair):

        return self.get_market_orders(pair)['bids']

    def get_market_trade_history(self, pair, since="hour"):
        """get market trade history; since {minute, hour, day}"""

        pair = self.format_pair(pair)

        return self.api("transactions/" + pair + "/?time={0}".format(since))

    def get_market_volume(self, pair):
        '''return market volume [of last 24h]'''

        pair = self.format_pair(pair)

        return self.get_market_ticker(pair)['volume']

    ########################
    # Private methods bellow
    ########################

    def get_balances(self, coin=None):
        '''Returns the values relevant to the specified <coin> parameter.'''

        if not coin:
            return self.private_api("v2/balance/")
        else:
            return {k: v for k, v in self.private_api("v2/balance/").items() if k.startswith(coin.lower())}

    def get_deposit_address(self, coin=None):
        '''get deposit address'''

        if coin == 'btc':
            command = 'bitcoin_deposit_address/'
        if coin == 'ltc':
            command = 'v2/ltc_address/'
        if coin == 'eth':
            command = 'v2/eth_address/'
        if coin == 'xrp':
            command = 'ripple_address/'
        if coin == 'bch':
            command = 'v2/bch_address/'

        return self.private_api(command)

    def get_liquidation_address(self, fiat):
        '''Creates new liquidation address which will automatically sell your BTC for specified liquidation_currency.'''

        return self.private_api('v2/liquidation_address/new/',
                                data={'liquidation_currency': fiat.lower()})

    def get_liquidation_address_info(self, address=None):
        '''Shows transactions (BTC to liquidation_currency) for liquidation address.'''

        return self.private_api('v2/liquidation_address/info/',
                                data={'address': address})

    def buy_limit(self, pair, rate, amount, daily_order=False):
        '''submit limit buy order

        daily_order: opens buy limit order which will be canceled at 0:00 UTC unless it already has been executed. Possible value: True'''

        pair = self.format_pair(pair)
        return self.private_api('v2/buy/{}/'.format(pair),
                                data={'amount': amount,
                                      'price': rate,
                                      'daily_order': daily_order
                                      }
                                )

    def buy_market(self, pair, amount):
        '''submit market buy order'''

        pair = self.format_pair(pair)
        return self.private_api('v2/buy/market/{}/'.format(pair),
                                data={'amount': amount}
                                )

    def sell_limit(self, pair, rate, amount, daily_order=False):
        '''submit limit sell order'''

        pair = self.format_pair(pair)
        return self.private_api('v2/sell/{}/'.format(pair),
                                data={'amount': amount,
                                      'price': rate,
                                      'daily_order': daily_order
                                      }
                                )

    def sell_market(self, pair, amount):
        '''submit market sell order'''

        pair = self.format_pair(pair)
        return self.private_api('v2/sell/market/{}/'.format(pair),
                                data={'amount': amount})

    def cancel_order(self, order_id):
        '''cancel order by <order_id>'''

        return self.private_api('v2/cancel_order/', data={'id': order_id})

    def cancel_all_orders(self):
        '''cancel all active orders'''

        return self.private_api('cancel_all_orders/')

    def get_open_orders(self, pair):
        '''Get open orders.'''

        pair = self.format_pair(pair)
        return self.private_api("v2/open_orders/{}/".format(pair))

    def get_order(self, order_id):
        '''get order information'''

        return self.private_api('orders_status/', data={'id': order_id})

    def withdraw(self, coin, amount, address):
        '''withdraw cryptocurrency'''

        if coin == 'btc':
            command = 'bitcoin_withdrawal/'
        if coin == 'ltc':
            command = 'v2/ltc_withdrawal/'
        if coin == 'eth':
            command = 'v2/eth_withdrawal/'
        if coin == 'xrp':
            command = 'ripple_withdrawal/'
        if coin == 'bch':
            command = 'v2/bch_withdrawal/'

        return self.private_api(command, data={'amount': amount,
                                               'address': address}
                                )

    def get_transaction_history(self):
        '''Returns the history of transactions.'''

        raise NotImplementedError

    def get_deposit_history(self, coin=None):
        '''get deposit history'''

        raise NotImplementedError

    def get_withdraw_history(self, coin=None, timedelta=50000000):
        '''get withdrawals history'''

        return self.private_api('v2/withdrawal-requests/',
                                data={'timedelta': timedelta})

class BitstampNormalized(Bitstamp, NormalizedExchangeWrapper):

    def __init__(self, apikey=None, secret=None, customer_id=None, timeout=None, proxy=None):
        super(BitstampNormalized, self).__init__(apikey, secret, customer_id, timeout, proxy)

    @classmethod
    def format_pair(self, market_pair):
        """
        Expected input is quote - base.
        Normalize the pair inputs and
        format the pair argument to a format understood by the remote API."""

        market_pair = market_pair.lower()  # bittrex takes lowercase

        if "-" not in market_pair:
            raise InvalidDelimiterError('Agreed upon delimiter is "-".')

        quote, base = market_pair.split('-')

        if base not in self.base_currencies:
            raise InvalidBaseCurrencyError('''Expected input is quote-base, you have provided with {pair}'''.format(pair=market_pair))

        return quote + self.delimiter + base  # for bistamp quote comes first

    @staticmethod
    def _tstamp_to_datetime(timestamp):
        '''convert unix timestamp to datetime'''

        return datetime.fromtimestamp(timestamp)

    @staticmethod
    def _is_sale(s):

        if s == 0:
            return True
        else:
            return False

    def get_market_ticker(self, market):
        '''
        :return :
            dict['ask': float, 'bid': float, 'last': float]
            example: {'ask': float, 'bid': float, 'last': float}
        '''

        ticker = super().get_market_ticker(market)

        return {
            'ask': ticker['ask'],
            'bid': ticker['bid'],
            'last': ticker['last']
        }

    def get_market_trade_history(self, market):
        '''
        :return:
            list -> dict['timestamp': datetime.datetime,
                        'is_sale': bool,
                        'rate': float,
                        'amount': float,
                        'trade_id': any]
        '''

        upstream = super().get_market_trade_history(market)
        downstream = []

        for data in upstream:

            downstream.append({
                'timestamp': self._tstamp_to_datetime(int(data['date'])),
                'is_sale': self._is_sale(data['type']),
                'rate': data['price'],
                'amount': data['amount'],
                'trade_id': data['tid']
            })

        return downstream

    def get_market_orders(self, market):
        '''
        :return:
            dict['bids': list[price, quantity],
                 'asks': list[price, quantity]
                ]
        bids[0] should be first next to the spread
        asks[0] should be first next to the spread
        '''

        upstream = super().get_market_orders(market)

        return {
            'bids': [[i[0], i[1]] for i in upstream['bids']],
            'asks': [[i[0], i[1]] for i in upstream['asks']]
        }

    def get_market_spread(self, market):
        '''return first buy order and first sell order'''

        order_book = super().get_market_orders(market)

        ask = order_book['asks'][0][0]
        bid = order_book['bids'][0][0]

        return Decimal(ask) - Decimal(bid)

    def get_market_depth(self, market):
        '''return sum of all bids and asks'''

        order_book = self.get_market_orders(market)

        return {"bids": sum([Decimal(i[0]) * Decimal(i[1]) for i in order_book["bids"]]),
                "asks": sum([Decimal(i[1]) for i in order_book["asks"]])
                }
<|endoftext|>


<commit_before>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import asyncio
from concurrent.futures import ThreadPoolExecutor
from typing import Union, Tuple

# noinspection PyPackageRequirements
import grpc

from arch.api.proto import federation_pb2, federation_pb2_grpc
from arch.api.proto.federation_pb2 import TransferMeta, TransferDataDesc
from arch.api.transfer import Cleaner
from arch.api.transfer import Party, Federation
from arch.api.utils import eggroll_serdes
from arch.api.utils.log_utils import getLogger
from arch.api.utils.splitable import maybe_split_object, is_split_head, split_get
# noinspection PyProtectedMember
from eggroll.api.cluster.eggroll import _DTable, _EggRoll
from eggroll.api.proto import basic_meta_pb2, storage_basic_pb2

_ser_des = eggroll_serdes.PickleSerdes

OBJECT_STORAGE_NAME = "__federation__"

ERROR_STATES = [federation_pb2.CANCELLED, federation_pb2.ERROR]

LOGGER = getLogger()

async def _async_receive(stub, transfer_meta):
    resp_meta = stub.recv(transfer_meta)
    while resp_meta.transferStatus != federation_pb2.COMPLETE:
        if resp_meta.transferStatus in ERROR_STATES:
            raise IOError(
                "receive terminated, state: {}".format(federation_pb2.TransferStatus.Name(resp_meta.transferStatus)))
        resp_meta = stub.checkStatusNow(resp_meta)
        await asyncio.sleep(1)
    return resp_meta

def _thread_receive(receive_func, check_func, transfer_meta):
    LOGGER.debug(f"[GET] getting: transfer_meta={transfer_meta}")
    resp_meta = receive_func(transfer_meta)
    while resp_meta.transferStatus != federation_pb2.COMPLETE:
        if resp_meta.transferStatus in ERROR_STATES:
            raise IOError(
                "receive terminated, state: {}".format(federation_pb2.TransferStatus.Name(resp_meta.transferStatus)))
        resp_meta = check_func(resp_meta)
    LOGGER.debug(f"[GET] done: transfer_meta={transfer_meta}")
    return resp_meta

class FederationRuntime(Federation):

    def __init__(self, session_id, runtime_conf, host, port):
        super().__init__(session_id, runtime_conf)

        channel = grpc.insecure_channel(target=f"{host}:{port}",
                                        options=[('grpc.max_send_message_length', -1),
                                                 ('grpc.max_receive_message_length', -1)])
        self._stub = federation_pb2_grpc.TransferSubmitServiceStub(channel)
        self.__pool = ThreadPoolExecutor()
        self._cache_remote_obj_storage_table = {}
        self._cache_get_obj_storage_table = {}

    def remote(self, obj, name: str, tag: str, parties: Union[Party, list]) -> Cleaner:
        if isinstance(parties, Party):
            parties = [parties]
        self._remote_side_auth(name=name, parties=parties)
        cleaner = Cleaner()

        # if obj is a dtable, remote it
        if isinstance(obj, _DTable):
            obj.set_gc_disable()
            for party in parties:
                self._send(transfer_type=federation_pb2.DTABLE, name=name, tag=tag, dst_party=party, cleaner=cleaner,
                           table=obj)
            return cleaner

        # if obj is object, put it in specified dtable, then remote the dtable
        first, fragments = maybe_split_object(obj)
        for party in parties:
            obj_table = self._remote_side_obj_storage_table(dst_party=party)

            # remote object or remote fragment header
            self._send(transfer_type=federation_pb2.OBJECT, name=name, tag=tag, dst_party=party, cleaner=cleaner,
                       table=obj_table, obj=first)

            # remote fragments
            for fragment_index, fragment in fragments:
                self._send(transfer_type=federation_pb2.OBJECT, name=name, tag=self._fragment_tag(tag, fragment_index),
                           dst_party=party, cleaner=cleaner, table=obj_table, obj=fragment)
        return cleaner

    def get(self, name: str, tag: str, parties: Union[Party, list]) -> Tuple[list, Cleaner]:
        if isinstance(parties, Party):
            parties = [parties]
        self._get_side_auth(name=name, parties=parties)
        cleaner = Cleaner()

        # async get
        meta_futures = [(party, self._receive(name, tag, party)) for party in parties]
        # block here, todo: any improvements?
        received_metas = [(party, meta.result()) for party, meta in meta_futures]

        rtn = []
        for party, recv_meta in received_metas:
            desc = recv_meta.dataDesc
            if desc.transferDataType == federation_pb2.DTABLE:
                dest_table = self._get_table(recv_meta.dataDesc)
                src = recv_meta.src
                dst = recv_meta.dst
                LOGGER.debug(
                    f"[GET] Got remote table {dest_table} from {src.name} {src.partyId} to {dst.name} {dst.partyId}")
                cleaner.add_table(dest_table)
                rtn.append(dest_table)
            elif desc.transferDataType == federation_pb2.OBJECT:
                obj_table = self._get_side_obj_storage_table(src_party=party)
                __tagged_key = _ser_des.deserialize(desc.taggedVariableName)
                obj = obj_table.get(__tagged_key)

                if not is_split_head(obj):
                    cleaner.add_obj(obj_table, __tagged_key)
                    rtn.append(obj)
                else:
                    num_split = obj.num_split()
                    fragments = []
                    fragment_meta_futures = [self._receive(name, self._fragment_tag(tag, i), party) for i in
                                             range(num_split)]
                    fragment_metas = [meta.result() for meta in fragment_meta_futures]
                    for meta in fragment_metas:
                        __fragment_tagged_key = _ser_des.deserialize(meta.dataDesc.taggedVariableName)
                        fragments.append(obj_table.get(__fragment_tagged_key))
                        cleaner.add_obj(obj_table, __fragment_tagged_key)
                    rtn.append(split_get(fragments))
            else:
                raise IOError(f"unknown transfer type: {recv_meta.dataDesc.transferDataType}")
        return rtn, cleaner

    """internal utils"""

    @staticmethod
    def _fragment_tag(tag, idx):
        return f"{tag}.__frag__{idx}"

    def _create_storage_table(self, src_party, dst_party):
        name = f"{OBJECT_STORAGE_NAME}.{src_party.role}-{src_party.party_id}-{dst_party.role}-{dst_party.party_id}"
        return _EggRoll.get_instance().table(name=name, namespace=self._session_id)

    def _remote_side_obj_storage_table(self, dst_party: Party):
        if dst_party in self._cache_remote_obj_storage_table:
            return self._cache_remote_obj_storage_table[dst_party]

        table = self._create_storage_table(self.local_party, dst_party)
        self._cache_remote_obj_storage_table[dst_party] = table
        return table

    def _get_side_obj_storage_table(self, src_party: Party):
        if src_party in self._cache_get_obj_storage_table:
            return self._cache_get_obj_storage_table[src_party]

        table = self._create_storage_table(src_party, self.local_party)
        self._cache_get_obj_storage_table[src_party] = table
        return table

    @staticmethod
    def _get_table(transfer_data_desc):
        name = transfer_data_desc.storageLocator.name,
        namespace = transfer_data_desc.storageLocator.namespace,
        persistent = transfer_data_desc.storageLocator.type != storage_basic_pb2.IN_MEMORY
        return _EggRoll.get_instance().table(name=name, namespace=namespace, persistent=persistent)

    def _send(self, transfer_type, name: str, tag: str, dst_party: Party, cleaner: Cleaner, table: _DTable, obj=None):
        tagged_key = f"{name}-{tag}"
        LOGGER.debug(f"[REMOTE] sending: type={transfer_type}, table={table}, tagged_key={tagged_key}")
        if transfer_type == federation_pb2.OBJECT:
            table.put(tagged_key, obj)
            cleaner.add_obj(table, tagged_key)
        else:
            cleaner.add_table(table)
        data_desc = TransferDataDesc(transferDataType=transfer_type,
                                     storageLocator=self.get_storage_locator(table),
                                     taggedVariableName=_ser_des.serialize(tagged_key))
        job = basic_meta_pb2.Job(jobId=self._session_id, name=name)
        transfer_meta = TransferMeta(job=job, tag=tag, src=self.local_party.to_pb(), dst=dst_party.to_pb(),
                                     dataDesc=data_desc, type=federation_pb2.SEND)
        self._stub.send(transfer_meta)
        LOGGER.debug(f"[REMOTE] done: type={transfer_type}, table={table}, tagged_key={tagged_key}")

    def _receive(self, name, tag, party: Party):
        job = basic_meta_pb2.Job(jobId=self._session_id, name=name)
        transfer_meta = TransferMeta(job=job, tag=tag, src=party.to_pb(), dst=self.local_party.to_pb(),
                                     type=federation_pb2.RECV)
        return self.__pool.submit(_thread_receive, self._stub.recv, self._stub.checkStatus, transfer_meta)

    # noinspection PyProtectedMember
    @staticmethod
    def get_storage_locator(table):
        return storage_basic_pb2.StorageLocator(type=table._type,
                                                namespace=table._namespace,
                                                name=table._name,
                                                fragment=table._partitions)
<commit_msg>an none block version<commit_after>#
#  Copyright 2019 The FATE Authors. All Rights Reserved.
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
#  limitations under the License.
#

import asyncio
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Union, Tuple

# noinspection PyPackageRequirements
import grpc

from arch.api.proto import federation_pb2, federation_pb2_grpc
from arch.api.proto.federation_pb2 import TransferMeta, TransferDataDesc
from arch.api.transfer import Cleaner
from arch.api.transfer import Party, Federation
from arch.api.utils import eggroll_serdes
from arch.api.utils.log_utils import getLogger
from arch.api.utils.splitable import maybe_split_object, is_split_head, split_get
# noinspection PyProtectedMember
from eggroll.api.cluster.eggroll import _DTable, _EggRoll
from eggroll.api.proto import basic_meta_pb2, storage_basic_pb2

_ser_des = eggroll_serdes.PickleSerdes

OBJECT_STORAGE_NAME = "__federation__"

ERROR_STATES = [federation_pb2.CANCELLED, federation_pb2.ERROR]

LOGGER = getLogger()

async def _async_receive(stub, transfer_meta):
    resp_meta = stub.recv(transfer_meta)
    while resp_meta.transferStatus != federation_pb2.COMPLETE:
        if resp_meta.transferStatus in ERROR_STATES:
            raise IOError(
                "receive terminated, state: {}".format(federation_pb2.TransferStatus.Name(resp_meta.transferStatus)))
        resp_meta = stub.checkStatusNow(resp_meta)
        await asyncio.sleep(1)
    return resp_meta

def _thread_receive(receive_func, check_func, transfer_meta):
    LOGGER.debug(f"[GET] getting: transfer_meta={transfer_meta}")
    resp_meta = receive_func(transfer_meta)
    while resp_meta.transferStatus != federation_pb2.COMPLETE:
        if resp_meta.transferStatus in ERROR_STATES:
            raise IOError(
                "receive terminated, state: {}".format(federation_pb2.TransferStatus.Name(resp_meta.transferStatus)))
        resp_meta = check_func(resp_meta)
    LOGGER.debug(f"[GET] done: transfer_meta={transfer_meta}")
    return resp_meta

class FederationRuntime(Federation):

    def __init__(self, session_id, runtime_conf, host, port):
        super().__init__(session_id, runtime_conf)

        channel = grpc.insecure_channel(target=f"{host}:{port}",
                                        options=[('grpc.max_send_message_length', -1),
                                                 ('grpc.max_receive_message_length', -1)])
        self._stub = federation_pb2_grpc.TransferSubmitServiceStub(channel)
        self.__pool = ThreadPoolExecutor()
        self._cache_remote_obj_storage_table = {}
        self._cache_get_obj_storage_table = {}

    def remote(self, obj, name: str, tag: str, parties: Union[Party, list]) -> Cleaner:
        if isinstance(parties, Party):
            parties = [parties]
        self._remote_side_auth(name=name, parties=parties)
        cleaner = Cleaner()

        # if obj is a dtable, remote it
        if isinstance(obj, _DTable):
            obj.set_gc_disable()
            for party in parties:
                self._send(transfer_type=federation_pb2.DTABLE, name=name, tag=tag, dst_party=party, cleaner=cleaner,
                           table=obj)
            return cleaner

        # if obj is object, put it in specified dtable, then remote the dtable
        first, fragments = maybe_split_object(obj)
        for party in parties:
            obj_table = self._remote_side_obj_storage_table(dst_party=party)

            # remote object or remote fragment header
            self._send(transfer_type=federation_pb2.OBJECT, name=name, tag=tag, dst_party=party, cleaner=cleaner,
                       table=obj_table, obj=first)

            # remote fragments
            for fragment_index, fragment in fragments:
                self._send(transfer_type=federation_pb2.OBJECT, name=name, tag=self._fragment_tag(tag, fragment_index),
                           dst_party=party, cleaner=cleaner, table=obj_table, obj=fragment)
        return cleaner

    # todo: this block until all task complete, make it async
    def get(self, name: str, tag: str, parties: Union[Party, list]) -> Tuple[list, Cleaner]:
        if isinstance(parties, Party):
            parties = [parties]
        self._get_side_auth(name=name, parties=parties)
        cleaner = Cleaner()
        rtn = {}

        # async get
        meta_futures = {self._receive(name, tag, party): party for party in parties}
        for future in as_completed(meta_futures):
            party = meta_futures[future]
            recv_meta = future.result()
            desc = recv_meta.dataDesc
            if desc.transferDataType == federation_pb2.DTABLE:
                dest_table = self._get_table(recv_meta.dataDesc)
                cleaner.add_table(dest_table)
                rtn[party] = dest_table
            elif desc.transferDataType == federation_pb2.OBJECT:
                obj_table = self._get_side_obj_storage_table(src_party=party)
                __tagged_key = _ser_des.deserialize(desc.taggedVariableName)
                obj = obj_table.get(__tagged_key)

                if not is_split_head(obj):
                    cleaner.add_obj(obj_table, __tagged_key)
                    rtn[party] = obj
                else:
                    num_split = obj.num_split()
                    fragments = []
                    fragment_meta_futures = [self._receive(name, self._fragment_tag(tag, i), party) for i in
                                             range(num_split)]
                    fragment_metas = [meta.result() for meta in fragment_meta_futures]
                    for meta in fragment_metas:
                        __fragment_tagged_key = _ser_des.deserialize(meta.dataDesc.taggedVariableName)
                        fragments.append(obj_table.get(__fragment_tagged_key))
                        cleaner.add_obj(obj_table, __fragment_tagged_key)
                    rtn[party] = split_get(fragments)
            else:
                raise IOError(f"unknown transfer type: {recv_meta.dataDesc.transferDataType}")
        return [rtn[party] for party in parties], cleaner

    """internal utils"""

    @staticmethod
    def _fragment_tag(tag, idx):
        return f"{tag}.__frag__{idx}"

    def _create_storage_table(self, src_party, dst_party):
        name = f"{OBJECT_STORAGE_NAME}.{src_party.role}-{src_party.party_id}-{dst_party.role}-{dst_party.party_id}"
        return _EggRoll.get_instance().table(name=name, namespace=self._session_id)

    def _remote_side_obj_storage_table(self, dst_party: Party):
        if dst_party in self._cache_remote_obj_storage_table:
            return self._cache_remote_obj_storage_table[dst_party]

        table = self._create_storage_table(self.local_party, dst_party)
        self._cache_remote_obj_storage_table[dst_party] = table
        return table

    def _get_side_obj_storage_table(self, src_party: Party):
        if src_party in self._cache_get_obj_storage_table:
            return self._cache_get_obj_storage_table[src_party]

        table = self._create_storage_table(src_party, self.local_party)
        self._cache_get_obj_storage_table[src_party] = table
        return table

    @staticmethod
    def _get_table(transfer_data_desc):
        name = transfer_data_desc.storageLocator.name,
        namespace = transfer_data_desc.storageLocator.namespace,
        persistent = transfer_data_desc.storageLocator.type != storage_basic_pb2.IN_MEMORY
        return _EggRoll.get_instance().table(name=name, namespace=namespace, persistent=persistent)

    def _send(self, transfer_type, name: str, tag: str, dst_party: Party, cleaner: Cleaner, table: _DTable, obj=None):
        tagged_key = f"{name}-{tag}"
        LOGGER.debug(f"[REMOTE] sending: type={transfer_type}, table={table}, tagged_key={tagged_key}")
        if transfer_type == federation_pb2.OBJECT:
            table.put(tagged_key, obj)
            cleaner.add_obj(table, tagged_key)
        else:
            cleaner.add_table(table)
        data_desc = TransferDataDesc(transferDataType=transfer_type,
                                     storageLocator=self.get_storage_locator(table),
                                     taggedVariableName=_ser_des.serialize(tagged_key))
        job = basic_meta_pb2.Job(jobId=self._session_id, name=name)
        transfer_meta = TransferMeta(job=job, tag=tag, src=self.local_party.to_pb(), dst=dst_party.to_pb(),
                                     dataDesc=data_desc, type=federation_pb2.SEND)
        self._stub.send(transfer_meta)
        LOGGER.debug(f"[REMOTE] done: type={transfer_type}, table={table}, tagged_key={tagged_key}")

    def _receive(self, name, tag, party: Party):
        job = basic_meta_pb2.Job(jobId=self._session_id, name=name)
        transfer_meta = TransferMeta(job=job, tag=tag, src=party.to_pb(), dst=self.local_party.to_pb(),
                                     type=federation_pb2.RECV)
        return self.__pool.submit(_thread_receive, self._stub.recv, self._stub.checkStatus, transfer_meta)

    # noinspection PyProtectedMember
    @staticmethod
    def get_storage_locator(table):
        return storage_basic_pb2.StorageLocator(type=table._type,
                                                namespace=table._namespace,
                                                name=table._name,
                                                fragment=table._partitions)
<|endoftext|>


<commit_before>import logging
import coloredlogs
import sys
import os
from os.path import expanduser
import yaml

from son.workspace.project import Project

log = logging.getLogger(__name__)

class Workspace:

    WORKSPACE_VERSION = "0.01"

    DEFAULT_WORKSPACE_DIR = os.path.join(expanduser("~"), ".son-workspace")
    DEFAULT_SCHEMAS_DIR = os.path.join(expanduser("~"), ".son-schema")

    # Parameter strings for the configuration descriptor.
    CONFIG_STR_NAME = "name"
    CONFIG_STR_VERSION = "version"
    CONFIG_STR_CATALOGUES_DIR = "catalogues_dir"
    CONFIG_STR_CATALOGUE_NS_DIR = "ns_catalogue"
    CONFIG_STR_CATALOGUE_VNF_DIR = "vnf_catalogue"
    CONFIG_STR_CONFIG_DIR = "configuration_dir"
    CONFIG_STR_PLATFORMS_DIR = "platforms_dir"
    CONFIG_STR_PROJECTS_DIR = "projects_dir"
    CONFIG_STR_SCHEMAS_REMOTE_MASTER = "schemas_remote_master"
    CONFIG_STR_SCHEMAS_LOCAL_MASTER = "schemas_local_master"
    CONFIG_STR_DESCRIPTOR_EXTENSION = "descriptor_extension"
    CONFIG_STR_CATALOGUE_SERVERS = "catalogue_servers"
    CONFIG_STR_LOGGING_LEVEL = "log_level"

    __descriptor_name__ = "workspace.yml"

    def __init__(self, ws_root, ws_name='SONATA workspace', log_level='INFO'):
        self.log_level = log_level
        coloredlogs.install(level=log_level)
        self.ws_root = ws_root
        self.ws_name = ws_name
        self.dirs = dict()
        self.schemas = dict()
        self.descriptor_extension = ""
        self.load_default_config()
        # Catalogue servers
        self.catalogue_servers = []

    def load_default_config(self):
        self.dirs[self.CONFIG_STR_CATALOGUES_DIR] = 'catalogues'
        self.dirs[self.CONFIG_STR_CONFIG_DIR] = 'configuration'
        self.dirs[self.CONFIG_STR_PLATFORMS_DIR] = 'platforms'

        self.schemas[self.CONFIG_STR_SCHEMAS_LOCAL_MASTER] = Workspace.DEFAULT_SCHEMAS_DIR
        self.schemas[self.CONFIG_STR_SCHEMAS_REMOTE_MASTER] = \
            "https://raw.githubusercontent.com/sonata-nfv/son-schema/master/"

        # Sub-directories of catalogues
        self.dirs[self.CONFIG_STR_CATALOGUE_NS_DIR] = \
            os.path.join(self.dirs[self.CONFIG_STR_CATALOGUES_DIR], self.CONFIG_STR_CATALOGUE_NS_DIR)
        self.dirs[self.CONFIG_STR_CATALOGUE_VNF_DIR] = \
            os.path.join(self.dirs[self.CONFIG_STR_CATALOGUES_DIR], self.CONFIG_STR_CATALOGUE_VNF_DIR)

        # Projects dir (optional)
        self.dirs[self.CONFIG_STR_PROJECTS_DIR] = 'projects'

        # Extension for YAML - schema/descriptor files
        self.descriptor_extension = "yml"

    def create_dirs(self):
        """
        Create the base directory structure for the workspace
        Invoked upon workspace creation.
        :return:
        """

        log.info('Creating workspace at %s', self.ws_root)
        os.makedirs(self.ws_root, exist_ok=False)
        for d in self.dirs:
            path = os.path.join(self.ws_root, self.dirs[d])
            os.makedirs(path, exist_ok=True)

    def create_catalog_sample(self):
        d = {'name': 'My personal catalog',
             'credentials': 'personal'
             }

        ws_file_path = os.path.join(self.ws_root, self.dirs[self.CONFIG_STR_CATALOGUES_DIR], 'personal.yml')
        with open(ws_file_path, 'w') as ws_file:
            ws_file.write(yaml.dump(d, default_flow_style=False))

    def create_ws_descriptor(self):
        """
        Creates a workspace configuration file descriptor.
        This is triggered by workspace creation and configuration changes.
        :return:
        """
        d = {self.CONFIG_STR_VERSION: Workspace.WORKSPACE_VERSION,
             self.CONFIG_STR_NAME: self.ws_name,
             self.CONFIG_STR_CATALOGUES_DIR: self.dirs[self.CONFIG_STR_CATALOGUES_DIR],
             self.CONFIG_STR_CONFIG_DIR: self.dirs[self.CONFIG_STR_CONFIG_DIR],
             self.CONFIG_STR_PLATFORMS_DIR: self.dirs[self.CONFIG_STR_PLATFORMS_DIR],
             self.CONFIG_STR_SCHEMAS_LOCAL_MASTER: self.dirs[self.CONFIG_STR_SCHEMAS_LOCAL_MASTER],
             self.CONFIG_STR_SCHEMAS_REMOTE_MASTER: self.dirs[self.CONFIG_STR_SCHEMAS_REMOTE_MASTER],
             self.CONFIG_STR_CATALOGUE_SERVERS: self.catalogue_servers,
             self.CONFIG_STR_LOGGING_LEVEL: self.log_level,
             self.CONFIG_STR_DESCRIPTOR_EXTENSION: self.descriptor_extension
             }

        ws_file_path = os.path.join(self.ws_root, Workspace.__descriptor_name__)
        with open(ws_file_path, 'w') as ws_file:
            yaml.dump(d, ws_file, default_flow_style=False)

    def create_files(self):
        self.create_ws_descriptor()
        self.create_catalog_sample()

    def check_ws_exists(self):
        ws_file = os.path.join(self.ws_root, Workspace.__descriptor_name__)
        return os.path.exists(ws_file) or os.path.exists(self.ws_root)

    @staticmethod
    def __create_from_descriptor__(ws_root):
        """
        Creates a Workspace object based on a configuration descriptor
        :param ws_root: base path of the workspace
        :return: Workspace object
        """
        ws_filename = os.path.join(ws_root, Workspace.__descriptor_name__)
        if not os.path.isdir(ws_root) or not os.path.isfile(ws_filename):
            log.error("Unable to load workspace descriptor '{}'".format(ws_filename))
            return None

        ws_file = open(ws_filename)
        ws_config = yaml.load(ws_file)

        if not ws_config[Workspace.CONFIG_STR_VERSION] == Workspace.WORKSPACE_VERSION:
            log.warning("Reading a workspace configuration with a different version")

        ws = Workspace(ws_root, ws_name=ws_config[Workspace.CONFIG_STR_NAME],
                       log_level=ws_config[Workspace.CONFIG_STR_LOGGING_LEVEL])
        ws.dirs[Workspace.CONFIG_STR_CATALOGUES_DIR] = ws_config[Workspace.CONFIG_STR_CATALOGUES_DIR]
        ws.dirs[Workspace.CONFIG_STR_CATALOGUES_DIR] = ws_config[Workspace.CONFIG_STR_CONFIG_DIR]
        ws.dirs[Workspace.CONFIG_STR_CONFIG_DIR] = ws_config[Workspace.CONFIG_STR_CONFIG_DIR]
        ws.dirs[Workspace.CONFIG_STR_PLATFORMS_DIR] = ws_config[Workspace.CONFIG_STR_PLATFORMS_DIR]
        ws.dirs[Workspace.CONFIG_STR_SCHEMAS_LOCAL_MASTER] = \
            expanduser(ws_config[Workspace.CONFIG_STR_SCHEMAS_LOCAL_MASTER])
        ws.dirs[Workspace.CONFIG_STR_SCHEMAS_REMOTE_MASTER] = ws_config[Workspace.CONFIG_STR_SCHEMAS_REMOTE_MASTER]
        ws.catalogue_servers = ws_config[Workspace.CONFIG_STR_CATALOGUE_SERVERS]
        ws.descriptor_extension = ws_config[Workspace.CONFIG_STR_DESCRIPTOR_EXTENSION]

        return ws

def main():

    print(str(sys.argv))

    import argparse

    parser = argparse.ArgumentParser(description="Generate new sonata workspaces and project layouts")
    parser.add_argument("--init", help="Create a new sonata workspace", action="store_true")
    parser.add_argument("--workspace", help="location of existing (or new) workspace. If not specified "
                                            "will assume '{}'".format(Workspace.DEFAULT_WORKSPACE_DIR),
                        required=False)
    parser.add_argument("--project",
                        help="create a new project at the specified location", required=False)
    parser.add_argument("--debug", help="increases logging level to debug", required=False, action="store_true")

    args = parser.parse_args()

    log_level = "INFO"
    if args.debug:
        log_level = "DEBUG"
        coloredlogs.install(level=log_level)

    # Ensure that one argument is given (--init, --workspace or --project)
    if not args.init and not args.workspace and not args.project:
        parser.print_help()
        return

    # Ensure that argument --workspace is not alone
    if not args.init and args.workspace and not args.project:
        parser.print_help()
        return

    # If workspace arg is not given, create a workspace in user home
    if args.workspace is None:
        ws_root = Workspace.DEFAULT_WORKSPACE_DIR

        # If a workspace already exists at user home, throw an error and quit
        if args.init and os.path.isdir(ws_root):
            print("A workspace already exists in {}. Please specify a different location.\n"
                  .format(ws_root), file=sys.stderr)
            exit(1)

    else:
        ws_root = expanduser(args.workspace)

    if args.init:
        ws = Workspace(ws_root, log_level=log_level)
        if ws.check_ws_exists():
            print("A workspace already exists at the specified location, exiting", file=sys.stderr)
            exit(1)

        log.debug("Attempting to create a new workspace")
        cwd = os.getcwd()
        ws.create_dirs()
        ws.create_files()
        os.chdir(cwd)
        log.debug("Workspace created.")
    else:
        ws = Workspace.__create_from_descriptor__(ws_root)
        if not ws:
            print("Could not find a SONATA workspace at the specified location", file=sys.stderr)
            exit(1)

    if args.project is not None:
        log.debug("Attempting to create a new project")

        prj_root = os.path.expanduser(args.project)
        proj = Project(prj_root, ws)
        proj.create_prj()

        log.debug("Project created.")
<commit_msg>[Continued] Corrected a bug in workspace's location of schemas<commit_after>import logging
import coloredlogs
import sys
import os
from os.path import expanduser
import yaml

from son.workspace.project import Project

log = logging.getLogger(__name__)

class Workspace:

    WORKSPACE_VERSION = "0.01"

    DEFAULT_WORKSPACE_DIR = os.path.join(expanduser("~"), ".son-workspace")
    DEFAULT_SCHEMAS_DIR = os.path.join(expanduser("~"), ".son-schema")

    # Parameter strings for the configuration descriptor.
    CONFIG_STR_NAME = "name"
    CONFIG_STR_VERSION = "version"
    CONFIG_STR_CATALOGUES_DIR = "catalogues_dir"
    CONFIG_STR_CATALOGUE_NS_DIR = "ns_catalogue"
    CONFIG_STR_CATALOGUE_VNF_DIR = "vnf_catalogue"
    CONFIG_STR_CONFIG_DIR = "configuration_dir"
    CONFIG_STR_PLATFORMS_DIR = "platforms_dir"
    CONFIG_STR_PROJECTS_DIR = "projects_dir"
    CONFIG_STR_SCHEMAS_REMOTE_MASTER = "schemas_remote_master"
    CONFIG_STR_SCHEMAS_LOCAL_MASTER = "schemas_local_master"
    CONFIG_STR_DESCRIPTOR_EXTENSION = "descriptor_extension"
    CONFIG_STR_CATALOGUE_SERVERS = "catalogue_servers"
    CONFIG_STR_LOGGING_LEVEL = "log_level"

    __descriptor_name__ = "workspace.yml"

    def __init__(self, ws_root, ws_name='SONATA workspace', log_level='INFO'):
        self.log_level = log_level
        coloredlogs.install(level=log_level)
        self.ws_root = ws_root
        self.ws_name = ws_name
        self.dirs = dict()
        self.schemas = dict()
        self.descriptor_extension = ""
        self.load_default_config()
        # Catalogue servers
        self.catalogue_servers = []

    def load_default_config(self):
        self.dirs[self.CONFIG_STR_CATALOGUES_DIR] = 'catalogues'
        self.dirs[self.CONFIG_STR_CONFIG_DIR] = 'configuration'
        self.dirs[self.CONFIG_STR_PLATFORMS_DIR] = 'platforms'

        self.schemas[self.CONFIG_STR_SCHEMAS_LOCAL_MASTER] = Workspace.DEFAULT_SCHEMAS_DIR
        self.schemas[self.CONFIG_STR_SCHEMAS_REMOTE_MASTER] = \
            "https://raw.githubusercontent.com/sonata-nfv/son-schema/master/"

        # Sub-directories of catalogues
        self.dirs[self.CONFIG_STR_CATALOGUE_NS_DIR] = \
            os.path.join(self.dirs[self.CONFIG_STR_CATALOGUES_DIR], self.CONFIG_STR_CATALOGUE_NS_DIR)
        self.dirs[self.CONFIG_STR_CATALOGUE_VNF_DIR] = \
            os.path.join(self.dirs[self.CONFIG_STR_CATALOGUES_DIR], self.CONFIG_STR_CATALOGUE_VNF_DIR)

        # Projects dir (optional)
        self.dirs[self.CONFIG_STR_PROJECTS_DIR] = 'projects'

        # Extension for YAML - schema/descriptor files
        self.descriptor_extension = "yml"

    def create_dirs(self):
        """
        Create the base directory structure for the workspace
        Invoked upon workspace creation.
        :return:
        """

        log.info('Creating workspace at %s', self.ws_root)
        os.makedirs(self.ws_root, exist_ok=False)
        for d in self.dirs:
            path = os.path.join(self.ws_root, self.dirs[d])
            os.makedirs(path, exist_ok=True)

    def create_catalog_sample(self):
        d = {'name': 'My personal catalog',
             'credentials': 'personal'
             }

        ws_file_path = os.path.join(self.ws_root, self.dirs[self.CONFIG_STR_CATALOGUES_DIR], 'personal.yml')
        with open(ws_file_path, 'w') as ws_file:
            ws_file.write(yaml.dump(d, default_flow_style=False))

    def create_ws_descriptor(self):
        """
        Creates a workspace configuration file descriptor.
        This is triggered by workspace creation and configuration changes.
        :return:
        """
        d = {self.CONFIG_STR_VERSION: Workspace.WORKSPACE_VERSION,
             self.CONFIG_STR_NAME: self.ws_name,
             self.CONFIG_STR_CATALOGUES_DIR: self.dirs[self.CONFIG_STR_CATALOGUES_DIR],
             self.CONFIG_STR_CONFIG_DIR: self.dirs[self.CONFIG_STR_CONFIG_DIR],
             self.CONFIG_STR_PLATFORMS_DIR: self.dirs[self.CONFIG_STR_PLATFORMS_DIR],
             self.CONFIG_STR_SCHEMAS_LOCAL_MASTER: self.schemas[self.CONFIG_STR_SCHEMAS_LOCAL_MASTER],
             self.CONFIG_STR_SCHEMAS_REMOTE_MASTER: self.schemas[self.CONFIG_STR_SCHEMAS_REMOTE_MASTER],
             self.CONFIG_STR_CATALOGUE_SERVERS: self.catalogue_servers,
             self.CONFIG_STR_LOGGING_LEVEL: self.log_level,
             self.CONFIG_STR_DESCRIPTOR_EXTENSION: self.descriptor_extension
             }

        ws_file_path = os.path.join(self.ws_root, Workspace.__descriptor_name__)
        with open(ws_file_path, 'w') as ws_file:
            yaml.dump(d, ws_file, default_flow_style=False)

    def create_files(self):
        self.create_ws_descriptor()
        self.create_catalog_sample()

    def check_ws_exists(self):
        ws_file = os.path.join(self.ws_root, Workspace.__descriptor_name__)
        return os.path.exists(ws_file) or os.path.exists(self.ws_root)

    @staticmethod
    def __create_from_descriptor__(ws_root):
        """
        Creates a Workspace object based on a configuration descriptor
        :param ws_root: base path of the workspace
        :return: Workspace object
        """
        ws_filename = os.path.join(ws_root, Workspace.__descriptor_name__)
        if not os.path.isdir(ws_root) or not os.path.isfile(ws_filename):
            log.error("Unable to load workspace descriptor '{}'".format(ws_filename))
            return None

        ws_file = open(ws_filename)
        ws_config = yaml.load(ws_file)

        if not ws_config[Workspace.CONFIG_STR_VERSION] == Workspace.WORKSPACE_VERSION:
            log.warning("Reading a workspace configuration with a different version")

        ws = Workspace(ws_root, ws_name=ws_config[Workspace.CONFIG_STR_NAME],
                       log_level=ws_config[Workspace.CONFIG_STR_LOGGING_LEVEL])
        ws.dirs[Workspace.CONFIG_STR_CATALOGUES_DIR] = ws_config[Workspace.CONFIG_STR_CATALOGUES_DIR]
        ws.dirs[Workspace.CONFIG_STR_CATALOGUES_DIR] = ws_config[Workspace.CONFIG_STR_CONFIG_DIR]
        ws.dirs[Workspace.CONFIG_STR_CONFIG_DIR] = ws_config[Workspace.CONFIG_STR_CONFIG_DIR]
        ws.dirs[Workspace.CONFIG_STR_PLATFORMS_DIR] = ws_config[Workspace.CONFIG_STR_PLATFORMS_DIR]
        ws.schemas[Workspace.CONFIG_STR_SCHEMAS_LOCAL_MASTER] = \
            expanduser(ws_config[Workspace.CONFIG_STR_SCHEMAS_LOCAL_MASTER])
        ws.schemas[Workspace.CONFIG_STR_SCHEMAS_REMOTE_MASTER] = ws_config[Workspace.CONFIG_STR_SCHEMAS_REMOTE_MASTER]
        ws.catalogue_servers = ws_config[Workspace.CONFIG_STR_CATALOGUE_SERVERS]
        ws.descriptor_extension = ws_config[Workspace.CONFIG_STR_DESCRIPTOR_EXTENSION]

        return ws

def main():
    import argparse

    parser = argparse.ArgumentParser(description="Generate new sonata workspaces and project layouts")
    parser.add_argument("--init", help="Create a new sonata workspace", action="store_true")
    parser.add_argument("--workspace", help="location of existing (or new) workspace. If not specified "
                                            "will assume '{}'".format(Workspace.DEFAULT_WORKSPACE_DIR),
                        required=False)
    parser.add_argument("--project",
                        help="create a new project at the specified location", required=False)
    parser.add_argument("--debug", help="increases logging level to debug", required=False, action="store_true")

    args = parser.parse_args()

    log_level = "INFO"
    if args.debug:
        log_level = "DEBUG"
        coloredlogs.install(level=log_level)

    # Ensure that one argument is given (--init, --workspace or --project)
    if not args.init and not args.workspace and not args.project:
        parser.print_help()
        return

    # Ensure that argument --workspace is not alone
    if not args.init and args.workspace and not args.project:
        parser.print_help()
        return

    # If workspace arg is not given, create a workspace in user home
    if args.workspace is None:
        ws_root = Workspace.DEFAULT_WORKSPACE_DIR

        # If a workspace already exists at user home, throw an error and quit
        if args.init and os.path.isdir(ws_root):
            print("A workspace already exists in {}. Please specify a different location.\n"
                  .format(ws_root), file=sys.stderr)
            exit(1)

    else:
        ws_root = expanduser(args.workspace)

    if args.init:
        ws = Workspace(ws_root, log_level=log_level)
        if ws.check_ws_exists():
            print("A workspace already exists at the specified location, exiting", file=sys.stderr)
            exit(1)

        log.debug("Attempting to create a new workspace")
        cwd = os.getcwd()
        ws.create_dirs()
        ws.create_files()
        os.chdir(cwd)
        log.debug("Workspace created.")
    else:
        ws = Workspace.__create_from_descriptor__(ws_root)
        if not ws:
            print("Could not find a SONATA workspace at the specified location", file=sys.stderr)
            exit(1)

    if args.project is not None:
        log.debug("Attempting to create a new project")

        prj_root = os.path.expanduser(args.project)
        proj = Project(prj_root, ws)
        proj.create_prj()

        log.debug("Project created.")
<|endoftext|>


<commit_before>import sys
import os
import subprocess
import time
import csv
import urllib2
import pprint
import json

"""
Take status csv file from url and coordinate csv file to create
data dictionary format and convert that format into geojson type.
This geojson data can then be loaded into GitHub page to view
using it's mapping component

"""

"""
Input data strcture and output data structure (geojson)

Given remote rows from cso_status_data. Data when downloaded from url
11TH.CSOSTATUS_N,3
30TH.CSOSTATUS_N,3
3RD.CSOSTATUS_N,3

And local coords in the form of
[{'CSO_TagName': 'ALKI', 'X_COORD': '-122.4225', 'Y_COORD': '47.57024'},
 {'CSO_TagName': 'ALSK', 'X_COORD': '-122.40695', 'Y_COORD': '47.55944'},
 {'CSO_TagName': 'MURR', 'X_COORD': '-122.4', 'Y_COORD': '47.54028'},...]

formatted_geojson_data_dict = {'type':'FeatureCollection','features':
[{'type':'Feature','properties':{},'geometry':{'type':'Point','coordinates':[]}}]}

NEED a Data structure template in python to look like this then convert to  GeoJSON

{'type':'FeatureCollection",
  'features': [{'type': 'Features',
                'properties':{'CSO_TagName': 'ALKI',
                              'Value': 3},
                'geometries':{'type':'point',
                'coordinates':[-122.322,
                              47.607]}
                }
               ]
}

"""

# Downloading csv status values from the web, ftp site.
cso_status_data = urllib2.urlopen("http://your.kingcounty.gov/dnrp/library/wastewater/cso/img/CSO.CSV")

# Read csv file into a python list name cso_status_csv
text = cso_status_data.readlines()
cso_status_csv = csv.reader(text)
#pprint.pprint(cso_status_csv)

#Reading CSO with Coordinate in csv file locally and create list,
#subtitue with full data file cso_coord.csv or partial_coord.csv for two point data
cso_cord = open('cso_coord.csv', 'r')
reader = csv.DictReader(cso_cord)

location = list (reader)
cso_cord.close()
#pprint.pprint(location)

"""this the format we want to output
-question: not sure how to iterate the location object into below formatted_data_dict

formatted_geojson_data_dict = {'type':'FeatureCollection','features':
[{'type':'Feature','properties':{},'geometry':{'type':'Point','coordinates':[]}}]}

for row in location:
  formatted_geojson_data_dict['features'][row['CSO_TagName']] =
  		{'type':'Feature',
  		'properties':{},
  		'geometry':{'coordinates':[(row['X_COORD'])],[(row['Y_COORD'])]}}
 """

#Create dictionary with geojson template
geojson_data_dict = {'type':'FeatureCollection','features':[]}

for row in location:
    # We want to populate this stub, for every row, in the location list
    # {'type':'Features','properties':{},'geometry':{'type':'Point','coordinates':[]}}
    geojson_data_dict['features'].append({'type':'Feature',
                                          'properties':{'CSO_TagName':row['CSO_TagName'],
                                                        'DSN':row['DSN'],
                                                        'Name':row['Name'],
                                                        'Time_stamp':time.strftime("%Y-%m-%d %I:%M:%S", time.localtime()),
                                                        'Location(Lon/Lat)':"%1.3f , %1.3f" % (float(row["X_COORD"]) ,float(row["Y_COORD"])),
                                                        'CSO_Status':0,'marker-color':'#666',
                                                        'marker-size':'small',
                                                        'marker-symbol':'circle',
                                                        'description':'No Data Available'},
                                          'geometry':{'type':'Point',
                                                      'coordinates':[float(row["X_COORD"]), float(row["Y_COORD"])]
                                                     }
                                          })

#create brand new dictionary style with color according to that status
# Value 1 = #DC143C Discharging now
# Value 2 = #FFD700 Discharged last 48 hrs
# Value 3 = #00CD00 Not Discharging
# Value 4 = #0000EE No Real Time Data
#Did Not Work below
style_dict = {"1":{'marker-color':'#DC143C','marker-size':'small','description':'Discharging'},
              "2":{'marker-color':'#FFD700','marker-size':'small','description':'Discharge 48 hrs'},
              "3":{'marker-color':'#00CD00','marker-size':'small','description':'No Discharge'},
              "4":{'marker-color':'#0000EE','marker-size':'small','description':'No Real Time Data'}

              }

"""
style_dict = {"1":{'marker-color':'#DC143C'},
              "2":{'marker-color':'#FFD700'},
              "3":{'marker-color':'#00CD00'},
              "4":{'marker-color':'#0000EE'}}
"""

#??? - Not sure how to add value to be added onto geojson_data_dict object, replace with
##default vaue of 0........
"""with help from Paul, paul helped to crated loop to add CSO_Status value
(geojson_data_dict['features'][0]) is dict
and print it returns
{'geometry':{coordinates':[-122.4225,47.57024],'type':Point'},
'properties':{'CSO_Status':0,'CSO_TagName':'ALKI'},'type':'Feature'}

Replace geojson_data_dict's one of the value with CSO status. Refer to the note.

"""

# Populate with station values, based on station names.
for line in cso_status_csv:
    #Test to see record is  Seattle CSO data or not
    if line[0][0:5]=="NPDES": # this indicates the data is Seattle CSO
        cso_name = line[0]
        cso_symbol = 'circle-stroked'
    else: #this is not  Seattle CSO
        cso_name = line[0][0:len(line[0])-12] #this is for king county cso
        cso_symbol = 'Star Solid'
    #for all records
    CSO_Status = line[1]
    # If CSO exists, add to it.
    #Iterate through 'features' list
    for element in geojson_data_dict['features']:
      if cso_name == element['properties']['CSO_TagName']:
        element['properties']['CSO_Status'] = CSO_Status
        #element['properties'].append(style_dict[CSO_Status])
        element['properties']['marker-color']=style_dict[CSO_Status]['marker-color']
        element['properties']['marker-size']=style_dict[CSO_Status]['marker-size']
        element['properties']['description']=style_dict[CSO_Status]['description']
        #adding new element with symbol specific to seattle and KC
        element['properties']['marker-symbol']=cso_symbol

 #write out same element with additional style properties

formatted_geojson_data_dict = json.dumps(geojson_data_dict)
pprint.pprint(formatted_geojson_data_dict)

#take formatted_geojson_data_dict file and convert '' string into a file using with open
#out_file_fullpath ='/Users/peter/Documents/KC_CSO_Status/test_file5_5.geojson'

#take formatted_geojson_data_dict file and convert '' string into a file using with open down is for windows

#out_file_fullpath ='/Users/keump/Documents/GitHub/KC_CSO_Status/test_file.geojson' #for Windows 7

# file for public repo for Windows machine
out_file_fullpath = '/Users/keump/Documents/GitHub/data_display/cso_test_file.geojson'

# directory for public repo
out_file_fullpath_directory = '/Users/keump/Documents/GitHub/data_display'

# file for public repo for OS machine
#out_file_fullpath ='/Users/peter/Documents/KC_CSO_Status/test_file.geojson' #for macbook

with open(out_file_fullpath, 'w') as out_file:
   out_file.write(formatted_geojson_data_dict)

#using subprocess module to push the data into GitHub site to be view

subprocess.call(['git', '--git-dir', out_file_fullpath_directory + '/.git',
                '--work-tree', out_file_fullpath_directory,
                'add', out_file_fullpath])
subprocess.call(['git', '--git-dir', out_file_fullpath_directory  +'/.git',
                '--work-tree', out_file_fullpath_directory,
                'commit', '-a', '-m', '"Data Upload: ' + time.strftime("%Y-%m-%d %I:%M:%S", time.localtime()) + '"'])
subprocess.call(['git', '--git-dir', out_file_fullpath_directory + '/.git',
                '--work-tree', out_file_fullpath_directory,
                'push'])

<commit_msg>'update'<commit_after>import sys
import os
import subprocess
import time
import csv
import urllib2
import pprint
import json

"""
Take status csv file from url and coordinate csv file to create
data dictionary format and convert that format into geojson type.
This geojson data can then be loaded into GitHub page to view
using it's mapping component

"""

"""
Input data strcture and output data structure (geojson)

Given remote rows from cso_status_data. Data when downloaded from url
11TH.CSOSTATUS_N,3
30TH.CSOSTATUS_N,3
3RD.CSOSTATUS_N,3

And local coords in the form of
[{'CSO_TagName': 'ALKI', 'X_COORD': '-122.4225', 'Y_COORD': '47.57024'},
 {'CSO_TagName': 'ALSK', 'X_COORD': '-122.40695', 'Y_COORD': '47.55944'},
 {'CSO_TagName': 'MURR', 'X_COORD': '-122.4', 'Y_COORD': '47.54028'},...]

formatted_geojson_data_dict = {'type':'FeatureCollection','features':
[{'type':'Feature','properties':{},'geometry':{'type':'Point','coordinates':[]}}]}

NEED a Data structure template in python to look like this then convert to  GeoJSON

{'type':'FeatureCollection",
  'features': [{'type': 'Features',
                'properties':{'CSO_TagName': 'ALKI',
                              'Value': 3},
                'geometries':{'type':'point',
                'coordinates':[-122.322,
                              47.607]}
                }
               ]
}

"""

# Downloading csv status values from the web, ftp site.
cso_status_data = urllib2.urlopen("http://your.kingcounty.gov/dnrp/library/wastewater/cso/img/CSO.CSV")

# Read csv file into a python list name cso_status_csv
text = cso_status_data.readlines()
cso_status_csv = csv.reader(text)
#pprint.pprint(cso_status_csv)

#Reading CSO with Coordinate in csv file locally and create list,
#subtitue with full data file cso_coord.csv or partial_coord.csv for two point data
cso_cord = open('cso_coord.csv', 'r')
reader = csv.DictReader(cso_cord)

location = list (reader)
cso_cord.close()
#pprint.pprint(location)

"""this the format we want to output
-question: not sure how to iterate the location object into below formatted_data_dict

formatted_geojson_data_dict = {'type':'FeatureCollection','features':
[{'type':'Feature','properties':{},'geometry':{'type':'Point','coordinates':[]}}]}

for row in location:
  formatted_geojson_data_dict['features'][row['CSO_TagName']] =
  		{'type':'Feature',
  		'properties':{},
  		'geometry':{'coordinates':[(row['X_COORD'])],[(row['Y_COORD'])]}}
 """

#Create dictionary with geojson template
geojson_data_dict = {'type':'FeatureCollection','features':[]}

for row in location:
    # We want to populate this stub, for every row, in the location list
    # {'type':'Features','properties':{},'geometry':{'type':'Point','coordinates':[]}}
    geojson_data_dict['features'].append({'type':'Feature',
                                          'properties':{'CSO_TagName':row['CSO_TagName'],
                                                        'DSN':row['DSN'],
                                                        'Name':row['Name'],
                                                        'Time_stamp':time.strftime("%Y-%m-%d %I:%M:%S", time.localtime()),
                                                        'Location(Lon/Lat)':"%1.3f , %1.3f" % (float(row["X_COORD"]) ,float(row["Y_COORD"])),
                                                        'CSO_Status':0,'marker-color':'#666',
                                                        'marker-size':'small',
                                                        'marker-symbol':'circle',
                                                        'description':'No Data Available'},
                                          'geometry':{'type':'Point',
                                                      'coordinates':[float(row["X_COORD"]), float(row["Y_COORD"])]
                                                     }
                                          })

#create brand new dictionary style with color according to that status
# Value 1 = #DC143C Discharging now
# Value 2 = #FFD700 Discharged last 48 hrs
# Value 3 = #00CD00 Not Discharging
# Value 4 = #0000EE No Real Time Data
#Did Not Work below
style_dict = {"1":{'marker-color':'#DC143C','marker-size':'small','description':'Discharging'},
              "2":{'marker-color':'#FFD700','marker-size':'small','description':'Discharge 48 hrs'},
              "3":{'marker-color':'#00CD00','marker-size':'small','description':'No Discharge'},
              "4":{'marker-color':'#0000EE','marker-size':'small','description':'No Real Time Data'}

              }

"""
style_dict = {"1":{'marker-color':'#DC143C'},
              "2":{'marker-color':'#FFD700'},
              "3":{'marker-color':'#00CD00'},
              "4":{'marker-color':'#0000EE'}}
"""

#??? - Not sure how to add value to be added onto geojson_data_dict object, replace with
##default vaue of 0........
"""with help from Paul, paul helped to crated loop to add CSO_Status value
(geojson_data_dict['features'][0]) is dict
and print it returns
{'geometry':{coordinates':[-122.4225,47.57024],'type':Point'},
'properties':{'CSO_Status':0,'CSO_TagName':'ALKI'},'type':'Feature'}

Replace geojson_data_dict's one of the value with CSO status. Refer to the note.

"""

# Populate with station values, based on station names.
for line in cso_status_csv:
    #Test to see record is  Seattle CSO data or not
    if line[0][0:5]=="NPDES": # this indicates the data is Seattle CSO
        cso_name = line[0]
        cso_symbol = 'bus'
    else: #this is not  Seattle CSO
        cso_name = line[0][0:len(line[0])-12] #this is for king county cso
        cso_symbol = 'bus'
    #for all records
    CSO_Status = line[1]
    # If CSO exists, add to it.
    #Iterate through 'features' list
    for element in geojson_data_dict['features']:
      if cso_name == element['properties']['CSO_TagName']:
        element['properties']['CSO_Status'] = CSO_Status
        #element['properties'].append(style_dict[CSO_Status])
        element['properties']['marker-color']=style_dict[CSO_Status]['marker-color']
        element['properties']['marker-size']=style_dict[CSO_Status]['marker-size']
        element['properties']['description']=style_dict[CSO_Status]['description']
        #adding new element with symbol specific to seattle and KC
        element['properties']['marker-symbol']=cso_symbol

 #write out same element with additional style properties

formatted_geojson_data_dict = json.dumps(geojson_data_dict)
pprint.pprint(formatted_geojson_data_dict)

#take formatted_geojson_data_dict file and convert '' string into a file using with open
#out_file_fullpath ='/Users/peter/Documents/KC_CSO_Status/test_file5_5.geojson'

#take formatted_geojson_data_dict file and convert '' string into a file using with open down is for windows

#out_file_fullpath ='/Users/keump/Documents/GitHub/KC_CSO_Status/test_file.geojson' #for Windows 7

# file for public repo for Windows machine
out_file_fullpath = '/Users/keump/Documents/GitHub/data_display/cso_test_file.geojson'

# directory for public repo
out_file_fullpath_directory = '/Users/keump/Documents/GitHub/data_display'

# file for public repo for OS machine
#out_file_fullpath ='/Users/peter/Documents/KC_CSO_Status/test_file.geojson' #for macbook

with open(out_file_fullpath, 'w') as out_file:
   out_file.write(formatted_geojson_data_dict)

#using subprocess module to push the data into GitHub site to be view

subprocess.call(['git', '--git-dir', out_file_fullpath_directory + '/.git',
                '--work-tree', out_file_fullpath_directory,
                'add', out_file_fullpath])
subprocess.call(['git', '--git-dir', out_file_fullpath_directory  +'/.git',
                '--work-tree', out_file_fullpath_directory,
                'commit', '-a', '-m', '"Data Upload: ' + time.strftime("%Y-%m-%d %I:%M:%S", time.localtime()) + '"'])
subprocess.call(['git', '--git-dir', out_file_fullpath_directory + '/.git',
                '--work-tree', out_file_fullpath_directory,
                'push'])

<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
from optparse import OptionParser, OptionGroup

#-------------------------------------------------------------------------------

def parse_input(argv):

    parser = OptionParser(usage='./%prog --xy=example/xy.stick > example/output')

    group = OptionGroup(parser, 'Basic options')
    group.add_option('--xy',
                     action='store',
                     default=None,
                     help='Input data (energies and intensities of some sort)')
    group.add_option('--min',
                     type='float',
                     action='store',
                     default=1000.0,
                     help='Min x value [default: %default]')
    group.add_option('--max',
                     type='float',
                     action='store',
                     default=1800.0,
                     help='Max x value [default: %default]')
    group.add_option('--step',
                     type='float',
                     action='store',
                     default=1.0,
                     help='Stepsize in units of x values [default: %default]')
    group.add_option('--hwhm',
                     type='float',
                     action='store',
                     default=8.0,
                     help='Lorentzian half-width at half-maximum in units of x values [default: %default]')
    parser.add_option_group(group)

    if len(argv) == 1:
        # user has given no arguments and we are not in a test subdir: print help and exit
        print(parser.format_help().strip())
        sys.exit()

    return parser.parse_args()

#-------------------------------------------------------------------------------

def normalize(l):

    largest = 0.0
    for i in range(len(l)):
        if abs(l[i]) > largest:
            largest = abs(l[i])
    for i in range(len(l)):
        l[i] = l[i]/largest

    return l

#-------------------------------------------------------------------------------

def get_lorentzians(x_l, f_l, i_l, hwhm):

    y_l = []
    for i in range(len(x_l)):
        y_l.append(0.0)
    for i in range(len(x_l)):
        for j in range(len(f_l)):
            y_l[i] = y_l[i] + i_l[j]/(1 + (((x_l[i] - f_l[j])/hwhm)**2.0))

    return normalize(y_l)

#-------------------------------------------------------------------------------

def get_xy(options, x, y):

    # create list of x values
    x_l = []
    _x = options.min
    x_l.append(_x)
    while True:
        _x += options.step
        if _x > options.max:
            break
        else:
           x_l.append(_x)

    y_l = get_lorentzians(x_l, x, y, options.hwhm)

    return x_l, y_l

def extract_numbers(file_name):
    x = []
    y = []
    with open(file_name, 'r') as f:
        for line in f.readlines():
            x.append(float(line.split()[0]))
            y.append(float(line.split()[1]))
    return x, y

def main():

    (options, args) = parse_input(sys.argv)

    x, y = extract_numbers(options.xy)
    x_l, y_l = get_xy(options, x, y)

    # write lorentzians to stdout
    for i in range(len(x_l)):
        print("%f %f" % (x_l[i], y_l[i]))

#-------------------------------------------------------------------------------

if __name__ == '__main__':
    main()
<commit_msg>refactor<commit_after>#!/usr/bin/env python

import sys
from optparse import OptionParser, OptionGroup

#-------------------------------------------------------------------------------

def parse_input(argv):

    parser = OptionParser(usage='./%prog --xy=example/xy.stick > example/output')

    group = OptionGroup(parser, 'Basic options')
    group.add_option('--xy',
                     action='store',
                     default=None,
                     help='Input data (energies and intensities of some sort)')
    group.add_option('--min',
                     type='float',
                     action='store',
                     default=1000.0,
                     help='Min x value [default: %default]')
    group.add_option('--max',
                     type='float',
                     action='store',
                     default=1800.0,
                     help='Max x value [default: %default]')
    group.add_option('--step',
                     type='float',
                     action='store',
                     default=1.0,
                     help='Stepsize in units of x values [default: %default]')
    group.add_option('--hwhm',
                     type='float',
                     action='store',
                     default=8.0,
                     help='Lorentzian half-width at half-maximum in units of x values [default: %default]')
    parser.add_option_group(group)

    if len(argv) == 1:
        # user has given no arguments and we are not in a test subdir: print help and exit
        print(parser.format_help().strip())
        sys.exit()

    return parser.parse_args()

#-------------------------------------------------------------------------------

def normalize(l):
    largest = max(l)
    for i in range(len(l)):
        l[i] = l[i]/largest
    return l

#-------------------------------------------------------------------------------

def get_lorentzians(x_l, f_l, i_l, hwhm):

    y_l = []
    for i in range(len(x_l)):
        y_l.append(0.0)
    for i in range(len(x_l)):
        for j in range(len(f_l)):
            y_l[i] = y_l[i] + i_l[j]/(1 + (((x_l[i] - f_l[j])/hwhm)**2.0))

    return normalize(y_l)

#-------------------------------------------------------------------------------

def get_xy(options, x, y):

    # create list of x values
    x_l = []
    _x = options.min
    x_l.append(_x)
    while True:
        _x += options.step
        if _x > options.max:
            break
        else:
           x_l.append(_x)

    y_l = get_lorentzians(x_l, x, y, options.hwhm)

    return x_l, y_l

def extract_numbers(file_name):
    x = []
    y = []
    with open(file_name, 'r') as f:
        for line in f.readlines():
            x.append(float(line.split()[0]))
            y.append(float(line.split()[1]))
    return x, y

def main():

    (options, args) = parse_input(sys.argv)

    x, y = extract_numbers(options.xy)
    x_l, y_l = get_xy(options, x, y)

    # write lorentzians to stdout
    for i in range(len(x_l)):
        print("%f %f" % (x_l[i], y_l[i]))

#-------------------------------------------------------------------------------

if __name__ == '__main__':
    main()
<|endoftext|>