<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import datetime
import time
from .exchanges.poloniex import Poloniex as PoloniexApi
from .chart import Chart

def get_market_name(market):
    return market[0]

def add_fee(btc, fee=0.025):
    return btc - (btc / 100 * fee)

class Coin(object):

    """Docstring for Coin."""

    def __init__(self, name, quantity, btc_value=None):
        self.name = name
        self.quantity = quantity
        self.btc_value = btc_value

    @property
    def value(self):
        return self.btc_value

class Market(object):

    """Docstring for Market. """

    def __init__(self, exchange, name):
        """TODO: to be defined1.

        :name: TODO

        """
        self._exchange = exchange
        self._name = name

    @property
    def url(self):
        return "{}{}".format(self._exchange.url, self._name)

    def _get_chart_data(self, resolution, timeframe):
        return self._exchange._api.chart(self._name,
                                         self._exchange.resolution2seconds(resolution),
                                         self._exchange.timeframe2seconds(timeframe))

    def get_chart(self, resolution="30m", timeframe="1d"):
        data = self._get_chart_data(resolution, timeframe)
        return Chart(data)

    def buy(self, btc, price=None, option=None):
        """Will buy coins on the market for the given amount of BTC. On
        default we will make a market order which means we will try to
        buy for the best price available. If price is given the order
        will be placed for at the given price. You can optionally
        provide some options. See
        :class:`cointrader.exchanges.poloniex.api` for more details.

        :btc: Amount of BTC
        :price: Optionally price for which you want to buy
        :option: Optionally some buy options
        :returns: Dict witch details on the order.
        """
        if price is None:
            # Get best price on market.
            orderbook = self._exchange._api.book(self._name)
            asks = orderbook["asks"]  # Asks in the meaning of "Who wants to buy my coins?"
            best_offer = asks[-1]
            price = float(best_offer[0])
            amount = btc / price
        # return  self._exchange._api.buy(self._name, amount, price, option)
        return {u'orderNumber': u'77875861209', u'resultingTrades': [{u'tradeID': u'3070199', u'rate': u'0.03370000', u'amount': u'0.00588054', u'date': u'2017-03-03 09:33:48', u'total': u'0.00019817', u'type': u'buy'}]}

    def sell(self, amount, price=None):
        if price is None:
            # Get best price on market.
            orderbook = self._exchange._api.book(self._name)
            bids = orderbook["bids"]  # Asks in the meaning of "Who wants to buy my coins?"
            best_offer = bids[-1]
            price = float(best_offer[0])
        # result self._exchange._api.sell(self._name, amount, price, option)
        return {u'orderNumber': u'77875861209', u'resultingTrades': [{u'tradeID': u'3070199', u'rate': u'0.03370000', u'amount': u'0.00588054', u'date': u'2017-03-03 09:33:48', u'total': u'0.00019817', u'type': u'buy'}]}

class BacktestMarket(Market):

    """Market to enable backtesting a strategy on the market."""

    def __init__(self, exchange, name):
        """TODO: to be defined1.

        :exchange: TODO
        :name: TODO

        """
        Market.__init__(self, exchange, name)
        self._chart_data = None
        self._backtest_tick = 1

    def continue_backtest(self):
        self._backtest_tick += 1
        if self._chart_data and len(self._chart_data) >= self._backtest_tick:
            return True
        return False

    def get_chart(self, resolution="30m", timeframe="1d"):
        if self._chart_data is None:
            self._chart_data = self._get_chart_data(resolution, timeframe)
        return Chart(self._chart_data[0:self._backtest_tick])

    def buy(self, btc, price=None):
        price = float(self._chart_data[0:self._backtest_tick][-1]['close'])
        date = datetime.datetime.utcfromtimestamp(self._chart_data[0:self._backtest_tick][-1]['date'])
        btc = add_fee(btc)
        amount = btc / price
        return {u'orderNumber': u'{}'.format(int(time.time() * 1000)),
                u'resultingTrades': [
                    {u'tradeID': u'{}'.format(int(time.time() * 1000)),
                     u'rate': u'{}'.format(price),
                     u'amount': u'{}'.format(amount),
                     u'date': u'{}'.format(date),
                     u'total': u'{}'.format(btc),
                     u'type': u'buy'}]}

    def sell(self, amount, price=None):
        price = float(self._chart_data[0:self._backtest_tick][-1]['close'])
        date = datetime.datetime.utcfromtimestamp(self._chart_data[0:self._backtest_tick][-1]['date'])
        btc = add_fee(amount * price)
        return {u'orderNumber': u'{}'.format(int(time.time() * 1000)),
                u'resultingTrades': [
                    {u'tradeID': u'{}'.format(int(time.time() * 1000)),
                     u'rate': u'{}'.format(price),
                     u'amount': u'{}'.format(amount),
                     u'date': u'{}'.format(date),
                     u'total': u'{}'.format(btc),
                     u'type': u'sell'}]}

class Exchange(object):

    """Baseclass for all exchanges"""
    resolutions = {"5m": 5 * 60, "15m": 15 * 60,
                   "30m": 30 * 60, "1h": 60 * 60 * 1,
                   "2h": 60 * 60 * 2, "4h": 60 * 60 * 4, "24h": 60 * 60 * 24}
    timeframes = {"5m": 5 * 60, "15m": 15 * 60, "30m": 30 * 60,
                  "1h": 60 * 60, "2h": 60 * 60 * 2, "6h": 60 * 60 * 6,
                  "12h": 60 * 60 * 12, "1d": 60 * 60 * 24,
                  "2d": 60 * 60 * 24 * 2, "1w": 60 * 60 * 24 * 7,
                  "1M": 60 * 60 * 24 * 31, "3M": 60 * 60 * 24 * 31 * 3,
                  "1Y": 60 * 60 * 24 * 356}

    def __init__(self, config, api=None):
        """TODO: to be defined1. """
        self._api = api
        self.coins = {}

        # Setup coins
        balance = self._api.balance()
        for currency in balance:
            if balance[currency]["quantity"] > 0:
                self.coins[currency] = Coin(currency,
                                            balance[currency]["quantity"],
                                            balance[currency]["btc_value"])

    @property
    def url(self):
        raise NotImplementedError

    @property
    def total_btc_value(self):
        return sum([self.coins[c].value for c in self.coins])

    @property
    def total_euro_value(self, limit=10):
        ticker = self._api.ticker()
        return float(ticker["USDT_BTC"]["last"]) * self.total_btc_value

    @property
    def markets(self):
        ticker = self._api.ticker()
        tmp = {}
        for currency in ticker:
            if currency.startswith("BTC_"):
                change = round(float(ticker[currency]["percentChange"]) * 100, 2)
                volume = round(float(ticker[currency]["baseVolume"]), 1)
                if change <= 0:
                    continue
                tmp[currency] = {"volume": volume, "change": change}
        return tmp

    def get_top_markets(self, markets, limit=10):
        if not markets:
            markets = self.markets
        top_profit = self.get_top_profit_markets(markets, limit)
        top_volume = self.get_top_volume_markets(markets, limit)
        top_profit_markets = set(map(get_market_name, top_profit))
        top_volume_markets = set(map(get_market_name, top_volume))

        top_markets = {}
        for market in top_profit_markets.intersection(top_volume_markets):
            top_markets[market] = markets[market]
        return sorted(top_markets.items(), key=lambda x: x[1]["change"], reverse=True)[0:limit]

    def get_top_profit_markets(self, markets=None, limit=10):
        if not markets:
            markets = self.markets
        return sorted(markets.items(),
                      key=lambda x: (float(x[1]["change"]), float(x[1]["volume"])), reverse=True)[0:limit]

    def get_top_volume_markets(self, markets=None, limit=10):
        if not markets:
            markets = self.markets
        return sorted(markets.items(),
                      key=lambda x: (float(x[1]["volume"]), float(x[1]["change"])), reverse=True)[0:limit]

    def get_market(self, market, backtest=False):
        raise NotImplementedError

    def resolution2seconds(self, resolution):
        return self.resolutions[resolution]

    def timeframe2seconds(self, timeframe):
        return self.timeframes[timeframe]

class Poloniex(Exchange):

    def __init__(self, config):
        api = PoloniexApi(config)
        Exchange.__init__(self, config, api)

    @property
    def url(self):
        return "https://poloniex.com/exchange#"

    def btc2dollar(self, amount):
        ticker = self._api.ticker("USDT_BTC")
        rate = float(ticker["last"])
        return round(amount * rate, 2)

    def dollar2btc(self, amount):
        ticker = self._api.ticker("USDT_BTC")
        rate = float(ticker["last"])
        return round(amount / rate, 8)

    def get_balance(self, currency):
        return self._api.balance()[currency]

    def get_market(self, name, backtest=False):
        if backtest:
            return BacktestMarket(self, name)
        else:
            return Market(self, name)
<commit_msg>Make currency optional to return all balances.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import datetime
import time
from .exchanges.poloniex import Poloniex as PoloniexApi
from .chart import Chart

def get_market_name(market):
    return market[0]

def add_fee(btc, fee=0.025):
    return btc - (btc / 100 * fee)

class Coin(object):

    """Docstring for Coin."""

    def __init__(self, name, quantity, btc_value=None):
        self.name = name
        self.quantity = quantity
        self.btc_value = btc_value

    @property
    def value(self):
        return self.btc_value

class Market(object):

    """Docstring for Market. """

    def __init__(self, exchange, name):
        """TODO: to be defined1.

        :name: TODO

        """
        self._exchange = exchange
        self._name = name

    @property
    def url(self):
        return "{}{}".format(self._exchange.url, self._name)

    def _get_chart_data(self, resolution, timeframe):
        return self._exchange._api.chart(self._name,
                                         self._exchange.resolution2seconds(resolution),
                                         self._exchange.timeframe2seconds(timeframe))

    def get_chart(self, resolution="30m", timeframe="1d"):
        data = self._get_chart_data(resolution, timeframe)
        return Chart(data)

    def buy(self, btc, price=None, option=None):
        """Will buy coins on the market for the given amount of BTC. On
        default we will make a market order which means we will try to
        buy for the best price available. If price is given the order
        will be placed for at the given price. You can optionally
        provide some options. See
        :class:`cointrader.exchanges.poloniex.api` for more details.

        :btc: Amount of BTC
        :price: Optionally price for which you want to buy
        :option: Optionally some buy options
        :returns: Dict witch details on the order.
        """
        if price is None:
            # Get best price on market.
            orderbook = self._exchange._api.book(self._name)
            asks = orderbook["asks"]  # Asks in the meaning of "Who wants to buy my coins?"
            best_offer = asks[-1]
            price = float(best_offer[0])
            amount = btc / price
        # return  self._exchange._api.buy(self._name, amount, price, option)
        return {u'orderNumber': u'77875861209', u'resultingTrades': [{u'tradeID': u'3070199', u'rate': u'0.03370000', u'amount': u'0.00588054', u'date': u'2017-03-03 09:33:48', u'total': u'0.00019817', u'type': u'buy'}]}

    def sell(self, amount, price=None):
        if price is None:
            # Get best price on market.
            orderbook = self._exchange._api.book(self._name)
            bids = orderbook["bids"]  # Asks in the meaning of "Who wants to buy my coins?"
            best_offer = bids[-1]
            price = float(best_offer[0])
        # result self._exchange._api.sell(self._name, amount, price, option)
        return {u'orderNumber': u'77875861209', u'resultingTrades': [{u'tradeID': u'3070199', u'rate': u'0.03370000', u'amount': u'0.00588054', u'date': u'2017-03-03 09:33:48', u'total': u'0.00019817', u'type': u'buy'}]}

class BacktestMarket(Market):

    """Market to enable backtesting a strategy on the market."""

    def __init__(self, exchange, name):
        """TODO: to be defined1.

        :exchange: TODO
        :name: TODO

        """
        Market.__init__(self, exchange, name)
        self._chart_data = None
        self._backtest_tick = 1

    def continue_backtest(self):
        self._backtest_tick += 1
        if self._chart_data and len(self._chart_data) >= self._backtest_tick:
            return True
        return False

    def get_chart(self, resolution="30m", timeframe="1d"):
        if self._chart_data is None:
            self._chart_data = self._get_chart_data(resolution, timeframe)
        return Chart(self._chart_data[0:self._backtest_tick])

    def buy(self, btc, price=None):
        price = float(self._chart_data[0:self._backtest_tick][-1]['close'])
        date = datetime.datetime.utcfromtimestamp(self._chart_data[0:self._backtest_tick][-1]['date'])
        btc = add_fee(btc)
        amount = btc / price
        return {u'orderNumber': u'{}'.format(int(time.time() * 1000)),
                u'resultingTrades': [
                    {u'tradeID': u'{}'.format(int(time.time() * 1000)),
                     u'rate': u'{}'.format(price),
                     u'amount': u'{}'.format(amount),
                     u'date': u'{}'.format(date),
                     u'total': u'{}'.format(btc),
                     u'type': u'buy'}]}

    def sell(self, amount, price=None):
        price = float(self._chart_data[0:self._backtest_tick][-1]['close'])
        date = datetime.datetime.utcfromtimestamp(self._chart_data[0:self._backtest_tick][-1]['date'])
        btc = add_fee(amount * price)
        return {u'orderNumber': u'{}'.format(int(time.time() * 1000)),
                u'resultingTrades': [
                    {u'tradeID': u'{}'.format(int(time.time() * 1000)),
                     u'rate': u'{}'.format(price),
                     u'amount': u'{}'.format(amount),
                     u'date': u'{}'.format(date),
                     u'total': u'{}'.format(btc),
                     u'type': u'sell'}]}

class Exchange(object):

    """Baseclass for all exchanges"""
    resolutions = {"5m": 5 * 60, "15m": 15 * 60,
                   "30m": 30 * 60, "1h": 60 * 60 * 1,
                   "2h": 60 * 60 * 2, "4h": 60 * 60 * 4, "24h": 60 * 60 * 24}
    timeframes = {"5m": 5 * 60, "15m": 15 * 60, "30m": 30 * 60,
                  "1h": 60 * 60, "2h": 60 * 60 * 2, "6h": 60 * 60 * 6,
                  "12h": 60 * 60 * 12, "1d": 60 * 60 * 24,
                  "2d": 60 * 60 * 24 * 2, "1w": 60 * 60 * 24 * 7,
                  "1M": 60 * 60 * 24 * 31, "3M": 60 * 60 * 24 * 31 * 3,
                  "1Y": 60 * 60 * 24 * 356}

    def __init__(self, config, api=None):
        """TODO: to be defined1. """
        self._api = api
        self.coins = {}

        # Setup coins
        balance = self._api.balance()
        for currency in balance:
            if balance[currency]["quantity"] > 0:
                self.coins[currency] = Coin(currency,
                                            balance[currency]["quantity"],
                                            balance[currency]["btc_value"])

    @property
    def url(self):
        raise NotImplementedError

    @property
    def total_btc_value(self):
        return sum([self.coins[c].value for c in self.coins])

    @property
    def total_euro_value(self, limit=10):
        ticker = self._api.ticker()
        return float(ticker["USDT_BTC"]["last"]) * self.total_btc_value

    @property
    def markets(self):
        ticker = self._api.ticker()
        tmp = {}
        for currency in ticker:
            if currency.startswith("BTC_"):
                change = round(float(ticker[currency]["percentChange"]) * 100, 2)
                volume = round(float(ticker[currency]["baseVolume"]), 1)
                if change <= 0:
                    continue
                tmp[currency] = {"volume": volume, "change": change}
        return tmp

    def get_top_markets(self, markets, limit=10):
        if not markets:
            markets = self.markets
        top_profit = self.get_top_profit_markets(markets, limit)
        top_volume = self.get_top_volume_markets(markets, limit)
        top_profit_markets = set(map(get_market_name, top_profit))
        top_volume_markets = set(map(get_market_name, top_volume))

        top_markets = {}
        for market in top_profit_markets.intersection(top_volume_markets):
            top_markets[market] = markets[market]
        return sorted(top_markets.items(), key=lambda x: x[1]["change"], reverse=True)[0:limit]

    def get_top_profit_markets(self, markets=None, limit=10):
        if not markets:
            markets = self.markets
        return sorted(markets.items(),
                      key=lambda x: (float(x[1]["change"]), float(x[1]["volume"])), reverse=True)[0:limit]

    def get_top_volume_markets(self, markets=None, limit=10):
        if not markets:
            markets = self.markets
        return sorted(markets.items(),
                      key=lambda x: (float(x[1]["volume"]), float(x[1]["change"])), reverse=True)[0:limit]

    def get_market(self, market, backtest=False):
        raise NotImplementedError

    def resolution2seconds(self, resolution):
        return self.resolutions[resolution]

    def timeframe2seconds(self, timeframe):
        return self.timeframes[timeframe]

class Poloniex(Exchange):

    def __init__(self, config):
        api = PoloniexApi(config)
        Exchange.__init__(self, config, api)

    @property
    def url(self):
        return "https://poloniex.com/exchange#"

    def btc2dollar(self, amount):
        ticker = self._api.ticker("USDT_BTC")
        rate = float(ticker["last"])
        return round(amount * rate, 2)

    def dollar2btc(self, amount):
        ticker = self._api.ticker("USDT_BTC")
        rate = float(ticker["last"])
        return round(amount / rate, 8)

    def get_balance(self, currency=None):
        if currency is None:
            return self._api.balance()
        else:
            return self._api.balance()[currency]

    def get_market(self, name, backtest=False):
        if backtest:
            return BacktestMarket(self, name)
        else:
            return Market(self, name)
<|endoftext|>


<commit_before>
from spade import MTP
from spade import AID
from spade import ACLParser
import socket
import SocketServer
import xmpp

#try:
#	import stack_thread as thread
#except:
#	import thread

import thread

class simba(MTP.MTP):
	class SimbaRequestHandler(SocketServer.DatagramRequestHandler):
                        '''
                        Request handler for SIMBA messages
                        '''
                        def handle(self):
                                print "SIMBA SS: New incoming message: " + str(self.request)

	def receiveThread(self):
                self.SS = SocketServer.ThreadingUDPServer(("", self.port), SimbaRequestHandler)
                print "SIMBA SS listening on port ", self.port
                self.SS.serve_forever()

	def __setup(self):
		'''
		Secondary constructor
		'''
		self.address = self.config.acc[self.name].address
		self.port = self.config.acc[self.name].port

		# Launch receive thread
		print ">>>SIMBA: Going to start new thread"
		tid = thread.start_new_thread(self.receiveThread, ())
		print ">>>SIMBA: Started new thread " + str(tid)

	def send(self, envelope, payload, to=None):
		'''
		Send a message to a SIMBA agent
		'''
		print ">>>SIMBA TRANSPORT: A MESSAGE TO SEND FOR ME"

		payload = str(payload.getPayload()[0])
		print ">>>SIMBA: PAYLOAD = " + payload
		try:
			p = ACLParser.ACLxmlParser()
			aclmsg = p.parse(payload)
		except:
			print ">>>SIMBA: COULD NOT BUILD ACL"

		if to == None:
			to = envelope.getTo()
			print ">>>SIMBA TRANSPORT: TO = " + str(to)

		for receiver in to:
			print ">>>SIMBA TRANSPORT: RECEIVER = " + str(receiver)
			for ad in receiver.getAddresses():
				ad = str(ad)  # Type change
				print ">>>SIMBA TRANSPORT: ADDRESS = " + ad
				# SIMBA URI = simba://address:port
				if ad[0:8] == "simba://":
					ad = ad[8:]
					print ">>>SIMBA TRANSPORT: ADDRESS FINAL = " + ad
					# Check for the presence of a port
					if ':' in ad:
						ip, port = ad.split(':')
					else:
						ip = ad
						port = 2001

					# Set up a SIMBA socket to send the message
					s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
					try:
    						s.connect((ip, port))
						# FORCE ACL WITH PARENTHESIS
						s.send(str(aclmsg))  # ACL with parenthesis, oh no!
						s.close()
						print ">>>SIMBA message succesfully sent"
					except:
						print "Could not send SIMBA message"

# Required
PROTOCOL = "simba"
INSTANCE = simba
#port = "2001"
<commit_msg>packaging<commit_after>
from spade import MTP
from spade import AID
from spade import ACLParser
import socket
import SocketServer
import xmpp

#try:
#	import stack_thread as thread
#except:
#	import thread

import thread

class simba(MTP.MTP):
	class SimbaRequestHandler(SocketServer.DatagramRequestHandler):
                        '''
                        Request handler for SIMBA messages
                        '''
                        def handle(self):
                                print "SIMBA SS: New incoming message: " + str(self.request)

	def receiveThread(self):
                self.SS = SocketServer.ThreadingUDPServer(("", self.port), SimbaRequestHandler)
                print "SIMBA SS listening on port ", self.port
                self.SS.serve_forever()

	def __setup(self):
		'''
		Secondary constructor
		'''
		print ">>>SIMBA: __setup"
		self.address = self.config.acc[self.name].address
		self.port = self.config.acc[self.name].port

		# Launch receive thread
		print ">>>SIMBA: Going to start new thread"
		tid = thread.start_new_thread(self.receiveThread, ())
		print ">>>SIMBA: Started new thread " + str(tid)

	def send(self, envelope, payload, to=None):
		'''
		Send a message to a SIMBA agent
		'''
		print ">>>SIMBA TRANSPORT: A MESSAGE TO SEND FOR ME"

		payload = str(payload.getPayload()[0])
		print ">>>SIMBA: PAYLOAD = " + payload
		try:
			p = ACLParser.ACLxmlParser()
			aclmsg = p.parse(payload)
		except:
			print ">>>SIMBA: COULD NOT BUILD ACL"

		if to == None:
			to = envelope.getTo()
			print ">>>SIMBA TRANSPORT: TO = " + str(to)

		for receiver in to:
			print ">>>SIMBA TRANSPORT: RECEIVER = " + str(receiver)
			for ad in receiver.getAddresses():
				ad = str(ad)  # Type change
				print ">>>SIMBA TRANSPORT: ADDRESS = " + ad
				# SIMBA URI = simba://address:port
				if ad[0:8] == "simba://":
					ad = ad[8:]
					print ">>>SIMBA TRANSPORT: ADDRESS FINAL = " + ad
					# Check for the presence of a port
					if ':' in ad:
						ip, port = ad.split(':')
					else:
						ip = ad
						port = 2001

					# Set up a SIMBA socket to send the message
					s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
					try:
    						s.connect((ip, port))
						# FORCE ACL WITH PARENTHESIS
						s.send(str(aclmsg))  # ACL with parenthesis, oh no!
						s.close()
						print ">>>SIMBA message succesfully sent"
					except:
						print "Could not send SIMBA message"

# Required
PROTOCOL = "simba"
INSTANCE = simba
#port = "2001"
<|endoftext|>


<commit_before>#!/usr/bin/env python

__all__ = ['pptv_download', 'pptv_download_by_id']

from ..common import *

import re
import urllib
import hashlib

def pptv_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
    xml = get_html('http://web-play.pptv.com/webplay3-0-%s.xml?type=web.fpp' % id)
    host = r1(r'<sh>([^<>]+)</sh>', xml)
    key = r1(r'<key expire=[^<>]+>([^<>]+)</key>', xml)
    rid = r1(r'rid="([^"]+)"', xml)
    title = r1(r'nm="([^"]+)"', xml)
    pieces = re.findall('<sgm no="(\d+)".*fs="(\d+)"', xml)
    numbers, fs = zip(*pieces)
    urls = ['http://%s/%s/%s?k=%s' % (host, i, rid, key) for i in numbers]
    total_size = sum(map(int, fs))
    assert rid.endswith('.mp4')

    print_info(site_info, title, 'mp4', total_size)
    if not info_only:
        download_urls(urls, title, 'mp4', total_size, output_dir = output_dir, merge = merge)

def pptv_download(url, output_dir = '.', merge = True, info_only = False):
    assert re.match(r'http://v.pptv.com/show/(\w+)\.html$', url)
    html = get_html(url)
    id = r1(r'webcfg\s*=\s*{"id":\s*(\d+)', html)
    assert id
    pptv_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)

site_info = "PPTV.com"
download = pptv_download
download_playlist = playlist_not_supported('pptv')
<commit_msg>PPTV: fix multiple segments<commit_after>#!/usr/bin/env python

__all__ = ['pptv_download', 'pptv_download_by_id']

from ..common import *

import re
import urllib
import hashlib

def pptv_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
    xml = get_html('http://web-play.pptv.com/webplay3-0-%s.xml?type=web.fpp' % id)
    host = r1(r'<sh>([^<>]+)</sh>', xml)
    key = r1(r'<key expire=[^<>]+>([^<>]+)</key>', xml)
    rid = r1(r'rid="([^"]+)"', xml)
    title = r1(r'nm="([^"]+)"', xml)
    pieces = re.findall('<sgm no="(\d+)"[^<>]+fs="(\d+)"', xml)
    numbers, fs = zip(*pieces)
    urls = ['http://%s/%s/%s?k=%s' % (host, i, rid, key) for i in numbers]
    total_size = sum(map(int, fs))
    assert rid.endswith('.mp4')

    print_info(site_info, title, 'mp4', total_size)
    if not info_only:
        download_urls(urls, title, 'mp4', total_size, output_dir = output_dir, merge = merge)

def pptv_download(url, output_dir = '.', merge = True, info_only = False):
    assert re.match(r'http://v.pptv.com/show/(\w+)\.html$', url)
    html = get_html(url)
    id = r1(r'webcfg\s*=\s*{"id":\s*(\d+)', html)
    assert id
    pptv_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)

site_info = "PPTV.com"
download = pptv_download
download_playlist = playlist_not_supported('pptv')
<|endoftext|>


<commit_before>#!/usr/bin/python

import socket
import sys
import argparse

def main():
    parser = argparse.ArgumentParser(description = "Test a specified IP for open ports.")
    mutex = parser.add_mutually_exclusive_group()
    parser.add_argument('-v', '--verbose', help='Add extra verbosity to the output of the scanner', action='store_true')
    mutex.add_argument('-a', '--all', help='Scan all the possible ports', action='store_true')
    mutex.add_argument('-p', '--ports', help='Scan the specified ports only', type=int, metavar='PORT', choices=range(0,65536), nargs='*', default=[])

    args = parser.parse_args()

    if args.all:
        ports = range(0, 65536)
    elif args.ports:
        ports = args.ports
    else:
        ports = userPortInput()

    for port in ports:		    # For every given port attempt to connect...
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        banner = False
        try:
            if args.verbose:
                print "--> Attempting to connect to 127.0.0.1:" + str(port)
            s.connect(('127.0.0.1',int(port)))
            s.send("Port Checking")
            banner = s.recv(1024)
        except(socket.error):	    # If a socket.error exception is caught, it means the attempt to connect has failed,
            continue		    # hence the port is closed... In that case advance to the next port.
        if banner=='':
            banner = "No Response..."
        print "[+] Port "+ str(port) +" is open!"+ "  ==> Reply: "+ str(banner)
        s.close()

if __name__ =="__main__":
    main()
<commit_msg>Fix #2<commit_after>#!/usr/bin/python

import socket
import sys
import argparse

def main():
	parser = argparse.ArgumentParser(description = "Test a specified IP for open ports.")
	mutex = parser.add_mutually_exclusive_group()
	parser.add_argument('ip', metavar='IP', help='The ip to be scanned for open ports')
	parser.add_argument('-v', '--verbose', help='Add extra verbosity to the output of the scanner', action='store_true')
	mutex.add_argument('-a', '--all', help='Scan all the possible ports', action='store_true')
	mutex.add_argument('-p', '--ports', help='Scan the specified ports only', type=int, metavar='PORT', choices=range(0,65536), nargs='*', default=[])

	args = parser.parse_args()

	if args.all:
		ports = range(0, 65536)
	elif args.ports:
		ports = args.ports
	else:
		ports = userPortInput()

	ip = args.ip

	for port in ports:		    # For every given port attempt to connect...
		s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		banner = False
		try:
			if args.verbose:
				print "--> Attempting to connect to " + ip + ":" + str(port)
			s.connect((ip ,int(port)))
			s.send("Port Checking")
			banner = s.recv(1024)
		except(socket.error):	    # If a socket.error exception is caught, it means the attempt to connect has failed,
			continue		    # hence the port is closed... In that case advance to the next port.
		if banner=='':
			banner = "No Response..."
		print "[+] Port "+ str(port) +" is open!"+ "  ==> Reply: "+ str(banner)
		s.close()

if __name__ =="__main__":
	main()
<|endoftext|>


<commit_before># Copyright (c) 2010-2011 gocept gmbh & co. kg
# See also LICENSE.txt

import datetime
import gocept.runner
import logging
import pytz
import zeit.addcentral.interfaces
import zeit.brightcove.converter
import zeit.cms.content.interfaces
import zeit.cms.repository.interfaces
import zeit.cms.workflow.interfaces
import zeit.content.video.interfaces
import zope.component

log = logging.getLogger(__name__)

def update_from_brightcove():
    # getting the playlists seems to be much more likely to fail, and since
    # we want to take what we can get, we do it *after* we have processed
    # the videos (instead of combining both steps)
    VideoUpdater.update_all()
    PlaylistUpdater.update_all()

@gocept.runner.appmain(ticks=120, principal=gocept.runner.from_config(
    'zeit.brightcove', 'index-principal'))
def _update_from_brightcove():
    log.info('Update run started')
    update_from_brightcove()
    log.info('Update run finished')

class BaseUpdater(object):

    def __init__(self, context):
        log.debug('%r(%s)', self, context.uniqueId)
        self.bcobj = context
        self.cmsobj = zeit.cms.interfaces.ICMSContent(
            self.bcobj.uniqueId, None)
        log.debug('CMS object resolved: %r', self.cmsobj)

    def __call__(self):
        success = self.delete() or self.add() or self.update()
        if not success:
            log.warning('Object %s not processes.', self.bcobj.uniqueId)

    @classmethod
    def repository(self):
        return zope.component.getUtility(
            zeit.cms.repository.interfaces.IRepository)

    @classmethod
    def update_all(cls):
        for x in cls.get_objects():
            cls(x)()

    @classmethod
    def get_objects(cls):
        raise NotImplementedError()

    def add(self):
        if self.cmsobj is None:
            log.info('Adding %s', self.bcobj)
            log.debug('Getting add location')
            folder = zeit.addcentral.interfaces.IAddLocation(self.bcobj)
            log.debug('Adding ...')
            folder[str(self.bcobj.id)] = self.bcobj.to_cms()
            self.cmsobj = folder[str(self.bcobj.id)]
            log.debug('Create publish job')
            self._publish_if_allowed()
            return True

    def delete(self):
        pass

    def update(self):
        pass

    def _publish_if_allowed(self):
        zeit.cms.workflow.interfaces.IPublish(self.cmsobj).publish()

class VideoUpdater(BaseUpdater):

    @classmethod
    def get_objects(cls):
        now = datetime.datetime.now(pytz.UTC)
        from_date = (datetime.datetime(now.year, now.month, now.day, now.hour)
                     - datetime.timedelta(hours=10))
        return zeit.brightcove.converter.Video.find_modified(
            from_date=from_date)

    def delete(self):
        if self.bcobj.item_state == 'ACTIVE':
            # Handled elsewhere
            return False
        if self.bcobj.item_state == 'DELETED' and self.cmsobj is None:
            # Deleted in BC and no CMS object. We're done.
            return True
        if self.bcobj.item_state == 'INACTIVE' and self.cmsobj is None:
            # The item needs to be imported (but must not be published)
            return False
        if zeit.cms.workflow.interfaces.IPublishInfo(
                self.cmsobj).published:
            log.info('Retracting %s', self.bcobj)
            zeit.cms.workflow.interfaces.IPublish(self.cmsobj).retract()
        elif self.bcobj.item_state == 'DELETED':
            log.info('Deleting %s', self.bcobj)
            del self.cmsobj.__parent__[self.cmsobj.__name__]
        return True

    def update(self):
        # Update video in CMS iff the BC version is newer.
        new = self.bcobj.to_cms()
        update = True

        # A bug in Brightcove may cause the last-modified date to remain
        # unchanged even when the video-still URL is actually changed.
        # (Also note that Brightcove has the misfeature of sometimes returning
        # old data for several minutes even directly after we POSTed to update
        # something.)
        current_mtime = zeit.cms.content.interfaces.ISemanticChange(
            self.cmsobj).last_semantic_change
        new_mtime = zeit.cms.content.interfaces.ISemanticChange(
            new).last_semantic_change
        if (current_mtime and new_mtime and current_mtime >= new_mtime and
            self.cmsobj.video_still == new.video_still):
            update = False
        if update:
            log.info('Updating %s', self.bcobj)
            with zeit.cms.checkout.helper.checked_out(
                self.cmsobj, semantic_change=True, events=False) as co:
                # We don't need to send events here as a full checkout/checkin
                # cycle is done duing publication anyway.
                if co is None:
                    log.warning('Could not update video')
                else:
                    self.bcobj.to_cms(co)
        info = zeit.cms.workflow.interfaces.IPublishInfo(self.cmsobj)
        if update or not info.published:
            # If updated, publish in anycase; otherwise re-publish if not
            # published
            self._publish_if_allowed()
        return True

    def _publish_if_allowed(self):
        if self.bcobj.item_state == 'ACTIVE':
            zeit.cms.workflow.interfaces.IPublish(self.cmsobj).publish()

class PlaylistUpdater(BaseUpdater):

    @classmethod
    def get_objects(cls):
        return zeit.brightcove.converter.Playlist.find_all()

    @classmethod
    def update_all(cls):
        objects = cls.get_objects()
        for x in objects:
            cls(x)()
        cls.delete_remaining_except(objects)

    def update(self):
        current = self.bcobj.from_cms(self.cmsobj)
        update = False

        curdata = current.data
        newdata = self.bcobj.data
        for key in self.bcobj.fields.split(','):
            if key == 'id':
                continue
            if curdata.get(key) != newdata.get(key):
                update = True
                break

        if update:
            log.info('Updating %s', self.bcobj)
            with zeit.cms.checkout.helper.checked_out(self.cmsobj) as co:
                if co is None:
                    log.warning('Could not update playlist')
                else:
                    self.bcobj.to_cms(co)
        info = zeit.cms.workflow.interfaces.IPublishInfo(self.cmsobj)
        if update or not info.published:
            zeit.cms.workflow.interfaces.IPublish(self.cmsobj).publish()
        return True

    @classmethod
    def delete_remaining_except(cls, bc_objects):
        if not bc_objects:
            return
        folder = zeit.addcentral.interfaces.IAddLocation(
            iter(bc_objects).next())
        cms_names = set(folder.keys())
        bc_names = set(str(x.id) for x in bc_objects)
        for name in cms_names - bc_names:
            log.info('Deleting <Playlist id=%s>', name)
            cmsobj = folder[name]
            if zeit.cms.workflow.interfaces.IPublishInfo(cmsobj).published:
                zeit.cms.workflow.interfaces.IPublish(cmsobj).retract()
            del folder[name]
<commit_msg>use helper method to publish<commit_after># Copyright (c) 2010-2011 gocept gmbh & co. kg
# See also LICENSE.txt

import datetime
import gocept.runner
import logging
import pytz
import zeit.addcentral.interfaces
import zeit.brightcove.converter
import zeit.cms.content.interfaces
import zeit.cms.repository.interfaces
import zeit.cms.workflow.interfaces
import zeit.content.video.interfaces
import zope.component

log = logging.getLogger(__name__)

def update_from_brightcove():
    # getting the playlists seems to be much more likely to fail, and since
    # we want to take what we can get, we do it *after* we have processed
    # the videos (instead of combining both steps)
    VideoUpdater.update_all()
    PlaylistUpdater.update_all()

@gocept.runner.appmain(ticks=120, principal=gocept.runner.from_config(
    'zeit.brightcove', 'index-principal'))
def _update_from_brightcove():
    log.info('Update run started')
    update_from_brightcove()
    log.info('Update run finished')

class BaseUpdater(object):

    def __init__(self, context):
        log.debug('%r(%s)', self, context.uniqueId)
        self.bcobj = context
        self.cmsobj = zeit.cms.interfaces.ICMSContent(
            self.bcobj.uniqueId, None)
        log.debug('CMS object resolved: %r', self.cmsobj)

    def __call__(self):
        success = self.delete() or self.add() or self.update()
        if not success:
            log.warning('Object %s not processes.', self.bcobj.uniqueId)

    @classmethod
    def repository(self):
        return zope.component.getUtility(
            zeit.cms.repository.interfaces.IRepository)

    @classmethod
    def update_all(cls):
        for x in cls.get_objects():
            cls(x)()

    @classmethod
    def get_objects(cls):
        raise NotImplementedError()

    def add(self):
        if self.cmsobj is None:
            log.info('Adding %s', self.bcobj)
            log.debug('Getting add location')
            folder = zeit.addcentral.interfaces.IAddLocation(self.bcobj)
            log.debug('Adding ...')
            folder[str(self.bcobj.id)] = self.bcobj.to_cms()
            self.cmsobj = folder[str(self.bcobj.id)]
            log.debug('Create publish job')
            self._publish_if_allowed()
            return True

    def delete(self):
        pass

    def update(self):
        pass

    def _publish_if_allowed(self):
        zeit.cms.workflow.interfaces.IPublish(self.cmsobj).publish()

class VideoUpdater(BaseUpdater):

    @classmethod
    def get_objects(cls):
        now = datetime.datetime.now(pytz.UTC)
        from_date = (datetime.datetime(now.year, now.month, now.day, now.hour)
                     - datetime.timedelta(hours=10))
        return zeit.brightcove.converter.Video.find_modified(
            from_date=from_date)

    def delete(self):
        if self.bcobj.item_state == 'ACTIVE':
            # Handled elsewhere
            return False
        if self.bcobj.item_state == 'DELETED' and self.cmsobj is None:
            # Deleted in BC and no CMS object. We're done.
            return True
        if self.bcobj.item_state == 'INACTIVE' and self.cmsobj is None:
            # The item needs to be imported (but must not be published)
            return False
        if zeit.cms.workflow.interfaces.IPublishInfo(
                self.cmsobj).published:
            log.info('Retracting %s', self.bcobj)
            zeit.cms.workflow.interfaces.IPublish(self.cmsobj).retract()
        elif self.bcobj.item_state == 'DELETED':
            log.info('Deleting %s', self.bcobj)
            del self.cmsobj.__parent__[self.cmsobj.__name__]
        return True

    def update(self):
        # Update video in CMS iff the BC version is newer.
        new = self.bcobj.to_cms()
        update = True

        # A bug in Brightcove may cause the last-modified date to remain
        # unchanged even when the video-still URL is actually changed.
        # (Also note that Brightcove has the misfeature of sometimes returning
        # old data for several minutes even directly after we POSTed to update
        # something.)
        current_mtime = zeit.cms.content.interfaces.ISemanticChange(
            self.cmsobj).last_semantic_change
        new_mtime = zeit.cms.content.interfaces.ISemanticChange(
            new).last_semantic_change
        if (current_mtime and new_mtime and current_mtime >= new_mtime and
            self.cmsobj.video_still == new.video_still):
            update = False
        if update:
            log.info('Updating %s', self.bcobj)
            with zeit.cms.checkout.helper.checked_out(
                self.cmsobj, semantic_change=True, events=False) as co:
                # We don't need to send events here as a full checkout/checkin
                # cycle is done duing publication anyway.
                if co is None:
                    log.warning('Could not update video')
                else:
                    self.bcobj.to_cms(co)
        info = zeit.cms.workflow.interfaces.IPublishInfo(self.cmsobj)
        if update or not info.published:
            # If updated, publish in anycase; otherwise re-publish if not
            # published
            self._publish_if_allowed()
        return True

    def _publish_if_allowed(self):
        if self.bcobj.item_state == 'ACTIVE':
            zeit.cms.workflow.interfaces.IPublish(self.cmsobj).publish()

class PlaylistUpdater(BaseUpdater):

    @classmethod
    def get_objects(cls):
        return zeit.brightcove.converter.Playlist.find_all()

    @classmethod
    def update_all(cls):
        objects = cls.get_objects()
        for x in objects:
            cls(x)()
        cls.delete_remaining_except(objects)

    def update(self):
        current = self.bcobj.from_cms(self.cmsobj)
        update = False

        curdata = current.data
        newdata = self.bcobj.data
        for key in self.bcobj.fields.split(','):
            if key == 'id':
                continue
            if curdata.get(key) != newdata.get(key):
                update = True
                break

        if update:
            log.info('Updating %s', self.bcobj)
            with zeit.cms.checkout.helper.checked_out(self.cmsobj) as co:
                if co is None:
                    log.warning('Could not update playlist')
                else:
                    self.bcobj.to_cms(co)
        info = zeit.cms.workflow.interfaces.IPublishInfo(self.cmsobj)
        if update or not info.published:
            self._publish_if_allowed()
        return True

    @classmethod
    def delete_remaining_except(cls, bc_objects):
        if not bc_objects:
            return
        folder = zeit.addcentral.interfaces.IAddLocation(
            iter(bc_objects).next())
        cms_names = set(folder.keys())
        bc_names = set(str(x.id) for x in bc_objects)
        for name in cms_names - bc_names:
            log.info('Deleting <Playlist id=%s>', name)
            cmsobj = folder[name]
            if zeit.cms.workflow.interfaces.IPublishInfo(cmsobj).published:
                zeit.cms.workflow.interfaces.IPublish(cmsobj).retract()
            del folder[name]
<|endoftext|>


<commit_before>from urlparse import urlparse
from functools import wraps
import tempfile
import logging

from zinc.models import ZincIndex, ZincManifest, ZincCatalogConfig
from zinc.catalog import ZincAbstractCatalog, ZincCatalogPathHelper
from zinc.formats import Formats

from zinc.defaults import defaults
from zinc.utils import *
from zinc.helpers import *

log = logging.getLogger(__name__)

################################################################################

class ZincCatalog(ZincAbstractCatalog):

    def __init__(self, storage=None, coordinator=None, path_helper=None,
                 **kwargs):
        assert storage

        super(ZincCatalog, self).__init__(**kwargs)

        self._coordinator = coordinator
        self._storage = storage

        self._ph = path_helper or ZincCatalogPathHelper()
        self._manifests = {}
        self._reload()

        self.lock_timeout = defaults['catalog_lock_timeout']

    ### Properties ###

    @property
    def url(self):
        return self._storage.url

    @property
    def path(self):
        return urlparse(self.url).path

    @property
    def id(self):
        return self.index.id

    @property
    def path_helper(self):
        return self._ph

    def format(self):
        return self.index.format

    ### General Internal Methods ###

    def _reload(self):

        ## TODO: check format, just assume 1 for now
        self.index = self._read_index()
        if self.index.format != defaults['zinc_format']:
            raise Exception("Incompatible format %s" % (self.index.format))

        self._read_config_file()

    def _read_config_file(self):
        #log.warn('reimplement config loading')
        self.config = ZincCatalogConfig()
        #config_path = pjoin(self.path, defaults['catalog_config_name'])
        #self.config = load_config(config_path)

    def _lock_index(func):
        @wraps(func)
        def with_lock_index(self, *args, **kwargs):

            assert self._coordinator

            #lock = self._coordinator.get_index_lock_index()
            #lock.acquire(timeout=self.lock_timeout)
            #output = func(*args, **kwargs)
            #lock.release()

            with self._coordinator.get_index_lock(domain=self.id):
                self._reload()
                output = func(self, *args, **kwargs)
                self.save()

            return output
        return with_lock_index

    def save(self):
        self._write_index(self.index)

    ### I/O Helpers ###

    def _read(self, rel_path):
        f = self._storage.get(rel_path)
        return f.read() if f is not None else None

    def _write(self, subpath, bytes, raw=True, gzip=True):
        if raw:
            self._storage.puts(subpath, bytes)
        if gzip:
            self._storage.puts(subpath + '.gz', gzip_bytes(bytes))

    def _read_index(self):
        subpath = self._ph.path_for_index()
        bytes = self._read(subpath)
        return ZincIndex.from_bytes(bytes)

    def _write_index(self, index, raw=True, gzip=True):
        subpath = self._ph.path_for_index()
        bytes = index.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)
        if defaults['catalog_write_legacy_index']:
            self._write('index.json', bytes, raw=raw, gzip=gzip)

    def _read_manifest(self, bundle_name, version):
        subpath = self._ph.path_for_manifest_for_bundle_version(
                bundle_name, version)
        bytes = self._read(subpath)
        if bytes is not None:
            return ZincManifest.from_bytes(bytes)
        else:
            return None

    def _write_manifest(self, manifest, raw=True, gzip=True):
        subpath = self._ph.path_for_manifest(manifest)
        bytes = manifest.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)

    def _get_file_info(self, sha, preferred_formats=None):
        if preferred_formats is None:
            preferred_formats = defaults['catalog_preferred_formats']
        for format in preferred_formats:
            subpath = self._ph.path_for_file_with_sha(sha, format=format)
            meta = self._storage.get_meta(subpath)
            if meta is not None:
                return {
                        'sha' : sha,
                        'size' : meta['size'],
                        'format' : format
                        }
        return None

    def _read_file(self, sha, ext=None):
        subpath = self._ph.path_for_file_with_sha(sha, ext=ext)
        return self._storage.get(subpath)

    def _write_file(self, sha, src_path, format=None):
        format = format or Formats.RAW # default to RAW
        if format not in defaults['catalog_valid_formats']:
            raise Exception("Invalid format '%s'." % (format))
        ext = format if format != Formats.RAW else None
        subpath = self._ph.path_for_file_with_sha(sha, ext)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _get_archive_info(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(bundle_name,
                version, flavor=flavor)
        meta = self._storage.get_meta(subpath)
        return meta

    def _write_archive(self, bundle_name, version, src_path, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _read_archive(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        return self._storage.get(subpath)

    ### "Public" Methods

    def get_index(self):
        return self.index.clone(mutable=False)

    def get_manifest(self, bundle_name, version):
        return self._read_manifest(bundle_name, version)

    @_lock_index
    def update_bundle(self, new_manifest):

        assert new_manifest

        existing_versions = self.index.versions_for_bundle(new_manifest.bundle_name)
        if new_manifest.version in existing_versions:
            raise ValueError("Bundle version already exists.")

        next_version = self.index.next_version_for_bundle(new_manifest.bundle_name)
        if new_manifest.version > next_version:
            raise ValueError("Unexpected manifest version.")

        ## verify all files in the filelist exist in the repo

        missing_shas = list()
        info_by_path = dict()

        for path in new_manifest.files.keys():
            sha = new_manifest.sha_for_file(path)
            file_info = self._get_file_info(sha)
            if file_info is None:
                missing_shas.append(sha)
            else:
                info_by_path[path] = file_info

        if len(missing_shas) > 0:
            # TODO: better error
            raise Exception("Missing shas: %s" % (missing_shas))

        ## TODO: verify archives?

        ## write manifest

        self._write_manifest(new_manifest)

        ## update catalog index

        self.index.add_version_for_bundle(new_manifest.bundle_name,
                                          new_manifest.version)

    def import_path(self, src_path):

        sha = sha1_for_path(src_path)

        file_info = self._get_file_info(sha)
        if file_info is not None:
            return file_info

        # gzip the file first, and see if it passes the compression threshhold
        # TODO: this is stupid inefficient
        with tempfile.NamedTemporaryFile() as tmp_file:
            src_path_gz = tmp_file.name
            with open(src_path) as src_file:
                tmp_file.write(gzip_bytes(src_file.read()))
            tmp_file.flush()

            src_size = os.path.getsize(src_path)
            src_gz_size = os.path.getsize(src_path_gz)
            if src_size > 0 and float(src_gz_size) / src_size <= self.config.gzip_threshhold:
                final_src_path = src_path_gz
                final_src_size = src_gz_size
                format = Formats.GZ
            else:
                final_src_path = src_path
                final_src_size = src_size
                format = Formats.RAW

            imported_path = self._write_file(
                    sha, final_src_path, format=format)

        file_info = {
                'sha' :  sha,
                'size' : final_src_size,
                'format' : format
                }
        log.info("Imported %s --> %s" % (src_path, file_info))
        log.debug("Imported path: %s" % imported_path)
        return  file_info

    @_lock_index
    def _reserve_version_for_bundle(self, bundle_name):
        self.index.increment_next_version_for_bundle(bundle_name)
        return self.index.next_version_for_bundle(bundle_name)

    @_lock_index
    def delete_bundle_version(self, bundle_name, version):
        self.index.delete_bundle_version(bundle_name, version)

    @_lock_index
    def update_distribution(self, distribution_name, bundle_name, bundle_version):
        self.index.update_distribution(distribution_name, bundle_name, bundle_version)

    @_lock_index
    def delete_distribution(self, distribution_name, bundle_name):
        self.index.delete_distribution(distribution_name, bundle_name)

    @_lock_index
    def clean(self, dry_run=False):
        verb = 'Would remove' if dry_run else 'Removing'

        bundle_descriptors = self.bundle_descriptors()

        ### 1. scan manifests for ones that aren't in index

        dir = self._ph.manifests_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".json") or f.endswith(".json.gz")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        ### 2. scan archives for ones that aren't in index

        dir = self._ph.archives_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".tar")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        ### 3. clean objects

        all_objects = set()
        for bundle_desc in bundle_descriptors:
            manifest = self.manifest_for_bundle_descriptor(bundle_desc)
            for f, meta in manifest.files.iteritems():
                all_objects.add(meta['sha'])

        dir = self._ph.objects_dir
        for f in self._storage.list(dir):
            basename = os.path.splitext(f)[0]

            if basename not in all_objects:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

################################################################################

class ZincServiceProvider(object):
    pass

class ZincServiceConsumer(object):

    def __init__(self, **kwargs):
        pass

    def create_catalog(self, id=None, loc=None):
        raise NotImplementedError()

    def get_catalog(self, loc=None, id=None):
        raise NotImplementedError()

class CustomServiceConsumer(ZincServiceConsumer):

    def __init__(self, coordinator=None, storage=None, **kwargs):
        assert coordinator
        assert storage

        self._coordinator = coordinator
        self._storage = storage

    def get_catalog(self, loc=None, id=None):
        cat_storage = self._storage.bind_to_catalog(id=id)
        # TODO: bind to coordinator?
        return ZincCatalog(coordinator=self._coordinator, storage=cat_storage)
<commit_msg>Handle missing manifest<commit_after>from urlparse import urlparse
from functools import wraps
import tempfile
import logging

from zinc.models import ZincIndex, ZincManifest, ZincCatalogConfig
from zinc.catalog import ZincAbstractCatalog, ZincCatalogPathHelper
from zinc.formats import Formats

from zinc.defaults import defaults
from zinc.utils import *
from zinc.helpers import *

log = logging.getLogger(__name__)

################################################################################

class ZincCatalog(ZincAbstractCatalog):

    def __init__(self, storage=None, coordinator=None, path_helper=None,
                 **kwargs):
        assert storage

        super(ZincCatalog, self).__init__(**kwargs)

        self._coordinator = coordinator
        self._storage = storage

        self._ph = path_helper or ZincCatalogPathHelper()
        self._manifests = {}
        self._reload()

        self.lock_timeout = defaults['catalog_lock_timeout']

    ### Properties ###

    @property
    def url(self):
        return self._storage.url

    @property
    def path(self):
        return urlparse(self.url).path

    @property
    def id(self):
        return self.index.id

    @property
    def path_helper(self):
        return self._ph

    def format(self):
        return self.index.format

    ### General Internal Methods ###

    def _reload(self):

        ## TODO: check format, just assume 1 for now
        self.index = self._read_index()
        if self.index.format != defaults['zinc_format']:
            raise Exception("Incompatible format %s" % (self.index.format))

        self._read_config_file()

    def _read_config_file(self):
        #log.warn('reimplement config loading')
        self.config = ZincCatalogConfig()
        #config_path = pjoin(self.path, defaults['catalog_config_name'])
        #self.config = load_config(config_path)

    def _lock_index(func):
        @wraps(func)
        def with_lock_index(self, *args, **kwargs):

            assert self._coordinator

            #lock = self._coordinator.get_index_lock_index()
            #lock.acquire(timeout=self.lock_timeout)
            #output = func(*args, **kwargs)
            #lock.release()

            with self._coordinator.get_index_lock(domain=self.id):
                self._reload()
                output = func(self, *args, **kwargs)
                self.save()

            return output
        return with_lock_index

    def save(self):
        self._write_index(self.index)

    ### I/O Helpers ###

    def _read(self, rel_path):
        f = self._storage.get(rel_path)
        return f.read() if f is not None else None

    def _write(self, subpath, bytes, raw=True, gzip=True):
        if raw:
            self._storage.puts(subpath, bytes)
        if gzip:
            self._storage.puts(subpath + '.gz', gzip_bytes(bytes))

    def _read_index(self):
        subpath = self._ph.path_for_index()
        bytes = self._read(subpath)
        return ZincIndex.from_bytes(bytes)

    def _write_index(self, index, raw=True, gzip=True):
        subpath = self._ph.path_for_index()
        bytes = index.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)
        if defaults['catalog_write_legacy_index']:
            self._write('index.json', bytes, raw=raw, gzip=gzip)

    def _read_manifest(self, bundle_name, version):
        subpath = self._ph.path_for_manifest_for_bundle_version(
                bundle_name, version)
        bytes = self._read(subpath)
        if bytes is not None:
            return ZincManifest.from_bytes(bytes)
        else:
            return None

    def _write_manifest(self, manifest, raw=True, gzip=True):
        subpath = self._ph.path_for_manifest(manifest)
        bytes = manifest.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)

    def _get_file_info(self, sha, preferred_formats=None):
        if preferred_formats is None:
            preferred_formats = defaults['catalog_preferred_formats']
        for format in preferred_formats:
            subpath = self._ph.path_for_file_with_sha(sha, format=format)
            meta = self._storage.get_meta(subpath)
            if meta is not None:
                return {
                        'sha' : sha,
                        'size' : meta['size'],
                        'format' : format
                        }
        return None

    def _read_file(self, sha, ext=None):
        subpath = self._ph.path_for_file_with_sha(sha, ext=ext)
        return self._storage.get(subpath)

    def _write_file(self, sha, src_path, format=None):
        format = format or Formats.RAW # default to RAW
        if format not in defaults['catalog_valid_formats']:
            raise Exception("Invalid format '%s'." % (format))
        ext = format if format != Formats.RAW else None
        subpath = self._ph.path_for_file_with_sha(sha, ext)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _get_archive_info(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(bundle_name,
                version, flavor=flavor)
        meta = self._storage.get_meta(subpath)
        return meta

    def _write_archive(self, bundle_name, version, src_path, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _read_archive(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        return self._storage.get(subpath)

    ### "Public" Methods

    def get_index(self):
        return self.index.clone(mutable=False)

    def get_manifest(self, bundle_name, version):
        return self._read_manifest(bundle_name, version)

    @_lock_index
    def update_bundle(self, new_manifest):

        assert new_manifest

        existing_versions = self.index.versions_for_bundle(new_manifest.bundle_name)
        if new_manifest.version in existing_versions:
            raise ValueError("Bundle version already exists.")

        next_version = self.index.next_version_for_bundle(new_manifest.bundle_name)
        if new_manifest.version > next_version:
            raise ValueError("Unexpected manifest version.")

        ## verify all files in the filelist exist in the repo

        missing_shas = list()
        info_by_path = dict()

        for path in new_manifest.files.keys():
            sha = new_manifest.sha_for_file(path)
            file_info = self._get_file_info(sha)
            if file_info is None:
                missing_shas.append(sha)
            else:
                info_by_path[path] = file_info

        if len(missing_shas) > 0:
            # TODO: better error
            raise Exception("Missing shas: %s" % (missing_shas))

        ## TODO: verify archives?

        ## write manifest

        self._write_manifest(new_manifest)

        ## update catalog index

        self.index.add_version_for_bundle(new_manifest.bundle_name,
                                          new_manifest.version)

    def import_path(self, src_path):

        sha = sha1_for_path(src_path)

        file_info = self._get_file_info(sha)
        if file_info is not None:
            return file_info

        # gzip the file first, and see if it passes the compression threshhold
        # TODO: this is stupid inefficient
        with tempfile.NamedTemporaryFile() as tmp_file:
            src_path_gz = tmp_file.name
            with open(src_path) as src_file:
                tmp_file.write(gzip_bytes(src_file.read()))
            tmp_file.flush()

            src_size = os.path.getsize(src_path)
            src_gz_size = os.path.getsize(src_path_gz)
            if src_size > 0 and float(src_gz_size) / src_size <= self.config.gzip_threshhold:
                final_src_path = src_path_gz
                final_src_size = src_gz_size
                format = Formats.GZ
            else:
                final_src_path = src_path
                final_src_size = src_size
                format = Formats.RAW

            imported_path = self._write_file(
                    sha, final_src_path, format=format)

        file_info = {
            'sha':  sha,
            'size': final_src_size,
            'format': format
        }
        log.info("Imported %s --> %s" % (src_path, file_info))
        log.debug("Imported path: %s" % imported_path)
        return  file_info

    @_lock_index
    def _reserve_version_for_bundle(self, bundle_name):
        self.index.increment_next_version_for_bundle(bundle_name)
        return self.index.next_version_for_bundle(bundle_name)

    @_lock_index
    def delete_bundle_version(self, bundle_name, version):
        self.index.delete_bundle_version(bundle_name, version)

    @_lock_index
    def update_distribution(self, distribution_name, bundle_name, bundle_version):
        self.index.update_distribution(distribution_name, bundle_name, bundle_version)

    @_lock_index
    def delete_distribution(self, distribution_name, bundle_name):
        self.index.delete_distribution(distribution_name, bundle_name)

    @_lock_index
    def clean(self, dry_run=False):
        verb = 'Would remove' if dry_run else 'Removing'

        bundle_descriptors = self.bundle_descriptors()

        ### 1. scan manifests for ones that aren't in index

        dir = self._ph.manifests_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".json") or f.endswith(".json.gz")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        ### 2. scan archives for ones that aren't in index

        dir = self._ph.archives_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".tar")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        ### 3. clean objects

        all_objects = set()
        for bundle_desc in bundle_descriptors:
            manifest = self.manifest_for_bundle_descriptor(bundle_desc)
            if manifest is not None:
                for f, meta in manifest.files.iteritems():
                    all_objects.add(meta['sha'])

        dir = self._ph.objects_dir
        for f in self._storage.list(dir):
            basename = os.path.splitext(f)[0]

            if basename not in all_objects:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

################################################################################

class ZincServiceProvider(object):
    pass

class ZincServiceConsumer(object):

    def __init__(self, **kwargs):
        pass

    def create_catalog(self, id=None, loc=None):
        raise NotImplementedError()

    def get_catalog(self, loc=None, id=None):
        raise NotImplementedError()

class CustomServiceConsumer(ZincServiceConsumer):

    def __init__(self, coordinator=None, storage=None, **kwargs):
        assert coordinator
        assert storage

        self._coordinator = coordinator
        self._storage = storage

    def get_catalog(self, loc=None, id=None):
        cat_storage = self._storage.bind_to_catalog(id=id)
        # TODO: bind to coordinator?
        return ZincCatalog(coordinator=self._coordinator, storage=cat_storage)
<|endoftext|>


<commit_before>from urlparse import urlparse
from functools import wraps
import tempfile
import logging

from zinc.models import ZincIndex, ZincManifest, ZincCatalogConfig
from zinc.catalog import ZincAbstractCatalog, ZincCatalogPathHelper
from zinc.formats import Formats

from zinc.defaults import defaults
from zinc.utils import *
from zinc.helpers import *

log = logging.getLogger(__name__)

################################################################################

class ZincCatalog(ZincAbstractCatalog):

    def __init__(self, storage=None, coordinator=None, path_helper=None,
                 **kwargs):
        assert storage

        super(ZincCatalog, self).__init__(**kwargs)

        self._coordinator = coordinator
        self._storage = storage

        self._ph = path_helper or ZincCatalogPathHelper()
        self._manifests = {}
        self._reload()

        self.lock_timeout = defaults['catalog_lock_timeout']

    ### Properties ###

    @property
    def url(self):
        return self._storage.url

    @property
    def path(self):
        return urlparse(self.url).path

    @property
    def id(self):
        return self.index.id

    @property
    def path_helper(self):
        return self._ph

    def format(self):
        return self.index.format

    ### General Internal Methods ###

    def _reload(self):

        ## TODO: check format, just assume 1 for now
        self.index = self._read_index()
        if self.index.format != defaults['zinc_format']:
            raise Exception("Incompatible format %s" % (self.index.format))

        self._read_config_file()

    def _read_config_file(self):
        #log.warn('reimplement config loading')
        self.config = ZincCatalogConfig()
        #config_path = pjoin(self.path, defaults['catalog_config_name'])
        #self.config = load_config(config_path)

    def _lock_index(func):
        @wraps(func)
        def with_lock_index(self, *args, **kwargs):

            assert self._coordinator

            #lock = self._coordinator.get_index_lock_index()
            #lock.acquire(timeout=self.lock_timeout)
            #output = func(*args, **kwargs)
            #lock.release()

            with self._coordinator.get_index_lock(domain=self.id):
                self._reload()
                output = func(self, *args, **kwargs)
                self.save()

            return output
        return with_lock_index

    def save(self):
        self._write_index(self.index)

    ### I/O Helpers ###

    def _read(self, rel_path):
        f = self._storage.get(rel_path)
        return f.read() if f is not None else None

    def _write(self, subpath, bytes, raw=True, gzip=True):
        if raw:
            self._storage.puts(subpath, bytes)
        if gzip:
            self._storage.puts(subpath + '.gz', gzip_bytes(bytes))

    def _read_index(self):
        subpath = self._ph.path_for_index()
        bytes = self._read(subpath)
        return ZincIndex.from_bytes(bytes)

    def _write_index(self, index, raw=True, gzip=True):
        subpath = self._ph.path_for_index()
        bytes = index.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)
        if defaults['catalog_write_legacy_index']:
            self._write('index.json', bytes, raw=raw, gzip=gzip)

    def _read_manifest(self, bundle_name, version):
        subpath = self._ph.path_for_manifest_for_bundle_version(
                bundle_name, version)
        bytes = self._read(subpath)
        if bytes is not None:
            return ZincManifest.from_bytes(bytes)
        else:
            return None

    def _write_manifest(self, manifest, raw=True, gzip=True):
        subpath = self._ph.path_for_manifest(manifest)
        bytes = manifest.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)

    def _get_file_info(self, sha, preferred_formats=None):
        if preferred_formats is None:
            preferred_formats = defaults['catalog_preferred_formats']
        for format in preferred_formats:
            subpath = self._ph.path_for_file_with_sha(sha, format=format)
            meta = self._storage.get_meta(subpath)
            if meta is not None:
                return {
                        'sha' : sha,
                        'size' : meta['size'],
                        'format' : format
                        }
        return None

    def _read_file(self, sha, ext=None):
        subpath = self._ph.path_for_file_with_sha(sha, ext=ext)
        return self._storage.get(subpath)

    def _write_file(self, sha, src_path, format=None):
        format = format or Formats.RAW # default to RAW
        if format not in defaults['catalog_valid_formats']:
            raise Exception("Invalid format '%s'." % (format))
        ext = format if format != Formats.RAW else None
        subpath = self._ph.path_for_file_with_sha(sha, ext)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _get_archive_info(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(bundle_name,
                version, flavor=flavor)
        meta = self._storage.get_meta(subpath)
        return meta

    def _write_archive(self, bundle_name, version, src_path, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _read_archive(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        return self._storage.get(subpath)

    ### "Public" Methods

    def get_index(self):
        return self.index.clone(mutable=False)

    def get_manifest(self, bundle_name, version):
        return self._read_manifest(bundle_name, version)

    @_lock_index
    def update_bundle(self, new_manifest):

        assert new_manifest

        existing_versions = self.index.versions_for_bundle(new_manifest.bundle_name)
        if new_manifest.version in existing_versions:
            raise ValueError("Bundle version already exists.")

        next_version = self.index.next_version_for_bundle(new_manifest.bundle_name)
        if new_manifest.version > next_version:
            raise ValueError("Unexpected manifest version.")

        ## verify all files in the filelist exist in the repo

        missing_shas = list()
        info_by_path = dict()

        for path in new_manifest.files.keys():
            sha = new_manifest.sha_for_file(path)
            file_info = self._get_file_info(sha)
            if file_info is None:
                missing_shas.append(sha)
            else:
                info_by_path[path] = file_info

        if len(missing_shas) > 0:
            # TODO: better error
            raise Exception("Missing shas: %s" % (missing_shas))

        ## TODO: verify archives?

        ## write manifest

        self._write_manifest(new_manifest)

        ## update catalog index

        self.index.add_version_for_bundle(new_manifest.bundle_name,
                                          new_manifest.version)

    def import_path(self, src_path):

        sha = sha1_for_path(src_path)

        file_info = self._get_file_info(sha)
        if file_info is not None:
            return file_info

        # gzip the file first, and see if it passes the compression threshhold
        # TODO: this is stupid inefficient
        with tempfile.NamedTemporaryFile() as tmp_file:
            src_path_gz = tmp_file.name
            with open(src_path) as src_file:
                tmp_file.write(gzip_bytes(src_file.read()))
            tmp_file.flush()

            src_size = os.path.getsize(src_path)
            src_gz_size = os.path.getsize(src_path_gz)
            if src_size > 0 and float(src_gz_size) / src_size <= self.config.gzip_threshhold:
                final_src_path = src_path_gz
                final_src_size = src_gz_size
                format = Formats.GZ
            else:
                final_src_path = src_path
                final_src_size = src_size
                format = Formats.RAW

            imported_path = self._write_file(
                    sha, final_src_path, format=format)

        file_info = {
            'sha':  sha,
            'size': final_src_size,
            'format': format
        }
        log.info("Imported %s --> %s" % (src_path, file_info))
        log.debug("Imported path: %s" % imported_path)
        return  file_info

    @_lock_index
    def _reserve_version_for_bundle(self, bundle_name):
        self.index.increment_next_version_for_bundle(bundle_name)
        return self.index.next_version_for_bundle(bundle_name)

    @_lock_index
    def delete_bundle_version(self, bundle_name, version):
        self.index.delete_bundle_version(bundle_name, version)

    @_lock_index
    def update_distribution(self, distribution_name, bundle_name, bundle_version):
        self.index.update_distribution(distribution_name, bundle_name, bundle_version)

    @_lock_index
    def delete_distribution(self, distribution_name, bundle_name):
        self.index.delete_distribution(distribution_name, bundle_name)

    @_lock_index
    def clean(self, dry_run=False):
        verb = 'Would remove' if dry_run else 'Removing'

        bundle_descriptors = self.bundle_descriptors()

        ### 1. scan manifests for ones that aren't in index

        dir = self._ph.manifests_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".json") or f.endswith(".json.gz")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        ### 2. scan archives for ones that aren't in index

        dir = self._ph.archives_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".tar")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        ### 3. clean objects

        all_objects = set()
        for bundle_desc in bundle_descriptors:
            manifest = self.manifest_for_bundle_descriptor(bundle_desc)
            if manifest is not None:
                for f, meta in manifest.files.iteritems():
                    all_objects.add(meta['sha'])

        dir = self._ph.objects_dir
        for f in self._storage.list(dir):
            basename = os.path.splitext(f)[0]

            if basename not in all_objects:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

################################################################################

class ZincServiceProvider(object):
    pass

class ZincServiceConsumer(object):

    def __init__(self, **kwargs):
        pass

    def create_catalog(self, id=None, loc=None):
        raise NotImplementedError()

    def get_catalog(self, loc=None, id=None):
        raise NotImplementedError()

class CustomServiceConsumer(ZincServiceConsumer):

    def __init__(self, coordinator=None, storage=None, **kwargs):
        assert coordinator
        assert storage

        self._coordinator = coordinator
        self._storage = storage

    def get_catalog(self, loc=None, id=None):
        cat_storage = self._storage.bind_to_catalog(id=id)
        # TODO: bind to coordinator?
        return ZincCatalog(coordinator=self._coordinator, storage=cat_storage)
<commit_msg>catalog:clean bug fix<commit_after>from urlparse import urlparse
from functools import wraps
import tempfile
import logging

from zinc.models import ZincIndex, ZincManifest, ZincCatalogConfig
from zinc.catalog import ZincAbstractCatalog, ZincCatalogPathHelper
from zinc.formats import Formats

from zinc.defaults import defaults
from zinc.utils import *
from zinc.helpers import *

log = logging.getLogger(__name__)

################################################################################

class ZincCatalog(ZincAbstractCatalog):

    def __init__(self, storage=None, coordinator=None, path_helper=None,
                 **kwargs):
        assert storage

        super(ZincCatalog, self).__init__(**kwargs)

        self._coordinator = coordinator
        self._storage = storage

        self._ph = path_helper or ZincCatalogPathHelper()
        self._manifests = {}
        self._reload()

        self.lock_timeout = defaults['catalog_lock_timeout']

    ### Properties ###

    @property
    def url(self):
        return self._storage.url

    @property
    def path(self):
        return urlparse(self.url).path

    @property
    def id(self):
        return self.index.id

    @property
    def path_helper(self):
        return self._ph

    def format(self):
        return self.index.format

    ### General Internal Methods ###

    def _reload(self):

        ## TODO: check format, just assume 1 for now
        self.index = self._read_index()
        if self.index.format != defaults['zinc_format']:
            raise Exception("Incompatible format %s" % (self.index.format))

        self._read_config_file()

    def _read_config_file(self):
        #log.warn('reimplement config loading')
        self.config = ZincCatalogConfig()
        #config_path = pjoin(self.path, defaults['catalog_config_name'])
        #self.config = load_config(config_path)

    def _lock_index(func):
        @wraps(func)
        def with_lock_index(self, *args, **kwargs):

            assert self._coordinator

            #lock = self._coordinator.get_index_lock_index()
            #lock.acquire(timeout=self.lock_timeout)
            #output = func(*args, **kwargs)
            #lock.release()

            with self._coordinator.get_index_lock(domain=self.id):
                self._reload()
                output = func(self, *args, **kwargs)
                self.save()

            return output
        return with_lock_index

    def save(self):
        self._write_index(self.index)

    ### I/O Helpers ###

    def _read(self, rel_path):
        f = self._storage.get(rel_path)
        return f.read() if f is not None else None

    def _write(self, subpath, bytes, raw=True, gzip=True):
        if raw:
            self._storage.puts(subpath, bytes)
        if gzip:
            self._storage.puts(subpath + '.gz', gzip_bytes(bytes))

    def _read_index(self):
        subpath = self._ph.path_for_index()
        bytes = self._read(subpath)
        return ZincIndex.from_bytes(bytes)

    def _write_index(self, index, raw=True, gzip=True):
        subpath = self._ph.path_for_index()
        bytes = index.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)
        if defaults['catalog_write_legacy_index']:
            self._write('index.json', bytes, raw=raw, gzip=gzip)

    def _read_manifest(self, bundle_name, version):
        subpath = self._ph.path_for_manifest_for_bundle_version(
                bundle_name, version)
        bytes = self._read(subpath)
        if bytes is not None:
            return ZincManifest.from_bytes(bytes)
        else:
            return None

    def _write_manifest(self, manifest, raw=True, gzip=True):
        subpath = self._ph.path_for_manifest(manifest)
        bytes = manifest.to_bytes()
        self._write(subpath, bytes, raw=raw, gzip=gzip)

    def _get_file_info(self, sha, preferred_formats=None):
        if preferred_formats is None:
            preferred_formats = defaults['catalog_preferred_formats']
        for format in preferred_formats:
            subpath = self._ph.path_for_file_with_sha(sha, format=format)
            meta = self._storage.get_meta(subpath)
            if meta is not None:
                return {
                        'sha' : sha,
                        'size' : meta['size'],
                        'format' : format
                        }
        return None

    def _read_file(self, sha, ext=None):
        subpath = self._ph.path_for_file_with_sha(sha, ext=ext)
        return self._storage.get(subpath)

    def _write_file(self, sha, src_path, format=None):
        format = format or Formats.RAW # default to RAW
        if format not in defaults['catalog_valid_formats']:
            raise Exception("Invalid format '%s'." % (format))
        ext = format if format != Formats.RAW else None
        subpath = self._ph.path_for_file_with_sha(sha, ext)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _get_archive_info(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(bundle_name,
                version, flavor=flavor)
        meta = self._storage.get_meta(subpath)
        return meta

    def _write_archive(self, bundle_name, version, src_path, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        with open(src_path, 'r') as src_file:
            self._storage.put(subpath, src_file)
        return subpath

    def _read_archive(self, bundle_name, version, flavor=None):
        subpath = self._ph.path_for_archive_for_bundle_version(
                bundle_name, version, flavor=flavor)
        return self._storage.get(subpath)

    ### "Public" Methods

    def get_index(self):
        return self.index.clone(mutable=False)

    def get_manifest(self, bundle_name, version):
        return self._read_manifest(bundle_name, version)

    @_lock_index
    def update_bundle(self, new_manifest):

        assert new_manifest

        existing_versions = self.index.versions_for_bundle(new_manifest.bundle_name)
        if new_manifest.version in existing_versions:
            raise ValueError("Bundle version already exists.")

        next_version = self.index.next_version_for_bundle(new_manifest.bundle_name)
        if new_manifest.version > next_version:
            raise ValueError("Unexpected manifest version.")

        ## verify all files in the filelist exist in the repo

        missing_shas = list()
        info_by_path = dict()

        for path in new_manifest.files.keys():
            sha = new_manifest.sha_for_file(path)
            file_info = self._get_file_info(sha)
            if file_info is None:
                missing_shas.append(sha)
            else:
                info_by_path[path] = file_info

        if len(missing_shas) > 0:
            # TODO: better error
            raise Exception("Missing shas: %s" % (missing_shas))

        ## TODO: verify archives?

        ## write manifest

        self._write_manifest(new_manifest)

        ## update catalog index

        self.index.add_version_for_bundle(new_manifest.bundle_name,
                                          new_manifest.version)

    def import_path(self, src_path):

        sha = sha1_for_path(src_path)

        file_info = self._get_file_info(sha)
        if file_info is not None:
            return file_info

        # gzip the file first, and see if it passes the compression threshhold
        # TODO: this is stupid inefficient
        with tempfile.NamedTemporaryFile() as tmp_file:
            src_path_gz = tmp_file.name
            with open(src_path) as src_file:
                tmp_file.write(gzip_bytes(src_file.read()))
            tmp_file.flush()

            src_size = os.path.getsize(src_path)
            src_gz_size = os.path.getsize(src_path_gz)
            if src_size > 0 and float(src_gz_size) / src_size <= self.config.gzip_threshhold:
                final_src_path = src_path_gz
                final_src_size = src_gz_size
                format = Formats.GZ
            else:
                final_src_path = src_path
                final_src_size = src_size
                format = Formats.RAW

            imported_path = self._write_file(
                    sha, final_src_path, format=format)

        file_info = {
            'sha':  sha,
            'size': final_src_size,
            'format': format
        }
        log.info("Imported %s --> %s" % (src_path, file_info))
        log.debug("Imported path: %s" % imported_path)
        return  file_info

    @_lock_index
    def _reserve_version_for_bundle(self, bundle_name):
        self.index.increment_next_version_for_bundle(bundle_name)
        return self.index.next_version_for_bundle(bundle_name)

    @_lock_index
    def delete_bundle_version(self, bundle_name, version):
        self.index.delete_bundle_version(bundle_name, version)

    @_lock_index
    def update_distribution(self, distribution_name, bundle_name, bundle_version):
        self.index.update_distribution(distribution_name, bundle_name, bundle_version)

    @_lock_index
    def delete_distribution(self, distribution_name, bundle_name):
        self.index.delete_distribution(distribution_name, bundle_name)

    @_lock_index
    def clean(self, dry_run=False):
        verb = 'Would remove' if dry_run else 'Removing'

        bundle_descriptors = self.bundle_descriptors()

        ### 1. scan manifests for ones that aren't in index

        dir = self._ph.manifests_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".json") or f.endswith(".json.gz")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        ### 2. scan archives for ones that aren't in index

        dir = self._ph.archives_dir
        for f in self._storage.list(dir):
            remove = False
            if not (f.endswith(".tar")):
                # remove stray files
                remove = True
            else:
                bundle_descr = f.split(".")[0]
                if bundle_descr not in bundle_descriptors:
                    remove = True

            if remove:
                subpath = os.path.join(dir, f)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

        ### 3. clean objects

        all_objects = set()
        for bundle_desc in bundle_descriptors:
            manifest = self.manifest_for_bundle_descriptor(bundle_desc)
            for f, meta in manifest.files.iteritems():
                all_objects.add(meta['sha'])

        dir = self._ph.objects_dir
        for path in self._storage.list(dir):

            basename = os.path.basename(path)
            obj = os.path.splitext(basename)[0]

            if obj not in all_objects:
                subpath = os.path.join(dir, path)
                log.info("%s %s" % (verb, subpath))
                if not dry_run:
                    self._storage.delete(subpath)

################################################################################

class ZincServiceProvider(object):
    pass

class ZincServiceConsumer(object):

    def __init__(self, **kwargs):
        pass

    def create_catalog(self, id=None, loc=None):
        raise NotImplementedError()

    def get_catalog(self, loc=None, id=None):
        raise NotImplementedError()

class CustomServiceConsumer(ZincServiceConsumer):

    def __init__(self, coordinator=None, storage=None, **kwargs):
        assert coordinator
        assert storage

        self._coordinator = coordinator
        self._storage = storage

    def get_catalog(self, loc=None, id=None):
        cat_storage = self._storage.bind_to_catalog(id=id)
        # TODO: bind to coordinator?
        return ZincCatalog(coordinator=self._coordinator, storage=cat_storage)
<|endoftext|>