<commit_before>from ..SymbolDownloader import SymbolDownloader
from ..symbols.Stock import Stock

class StockDownloader(SymbolDownloader):
    def __init__(self):
        SymbolDownloader.__init__(self, "Stock")

    def decodeSymbolsContainer(self, symbolsContainer):
        symbols = []
        for row in symbolsContainer:
            ticker = row.contents[0].string
            companyName = row.contents[1].string
            type = row.contents[3].string
            categoryName = row.contents[4].string
            categoryNr = 0
            if(categoryName != None):
                categoryNr = row.contents[4].a.get('href').split("/").pop().split(".")[0]
            exchange = row.contents[5].string

            symbols.append(Stock(ticker, companyName, exchange, categoryName, categoryNr))
        return symbols

    def getRowHeader(self):
        return SymbolDownloader.getRowHeader(self) + ["categoryName", "categoryNr"]
<commit_msg>categoryNr is now a number<commit_after>from ..SymbolDownloader import SymbolDownloader
from ..symbols.Stock import Stock

class StockDownloader(SymbolDownloader):
    def __init__(self):
        SymbolDownloader.__init__(self, "Stock")

    def decodeSymbolsContainer(self, symbolsContainer):
        symbols = []
        for row in symbolsContainer:
            ticker = row.contents[0].string
            companyName = row.contents[1].string
            type = row.contents[3].string
            categoryName = row.contents[4].string
            categoryNr = 0
            if(categoryName != None):
                categoryNr = int(row.contents[4].a.get('href').split("/").pop().split(".")[0])
            exchange = row.contents[5].string

            symbols.append(Stock(ticker, companyName, exchange, categoryName, categoryNr))
        return symbols

    def getRowHeader(self):
        return SymbolDownloader.getRowHeader(self) + ["categoryName", "categoryNr"]
<|endoftext|>


<commit_before>import random
import logging

from zephyrus.message import Message
from zephyrus.strategy import Strategy

class ZDTStrategy(Strategy):
    def configure(self, content):
        self.niter = content['niter']
        self.length = content['length']
        self.nevaluators = content.get('nevaluators', 1)

    def mainloop(self):
        if self.nevaluators == 1:
            self.mainloop_single()
        else:
            self.mainloop_dist()

    def mainloop_single(self):
        # TODO expecting configuration here?
        best_solution = None
        best_value = None
        for i in range(self.niter):
            logging.info("Strategy: progress {}/{}".format(i + 1, self.niter))
            solution = [random.random() for _ in range(self.length)]
            msg = self.messenger.build_evaluate_message(content={'id': None, 'data': solution})
            self.socket_send.send_string(str(msg))
            ans = Message.from_string(self.socket_receive.recv_string())
            logging.debug('Strategy: received {}'.format(str(ans)))
            value = ans.content['data']
            if ans.type == 'RESULT':
                if best_value is None or best_value > value:
                    best_value = value
                    best_solution = solution
            elif ans.type == 'STOP':
                logging.info('Strategy: stopping.')
                break
        logging.debug('Strategy: best found {}'.format(best_value))
        logging.debug('Strategy: best solution {}'.format(best_solution))
        self.socket_send.send_string(str(self.messenger.build_stop_message()))
        msg = self.messenger.build_result_message(content={
            'value': best_value,
            'solution': best_solution
        })
        logging.debug('Strategy: sending result {}'.format(str(msg)))
        self.socket_send.send_string(str(msg))

    def mainloop_dist(self):
        # TODO expecting configuration here?
        best_solution = None
        best_value = None

        poller = zmq.Poller()
        poller.register(self.socket_receive)
        navailable = self.nevaluators
        nwaiting = 0
        tsent = 0
        trecv = 0
        while trecv < self.niter:
            while navailable > 0 and tsent < self.niter:
                logging.info("Strategy: progress {}/{}".format(i + 1, self.niter))
                solution = [random.random() for _ in range(self.length)]
                msg = self.messenger.build_evaluate_message(content=solution)
                self.socket_send.send_string(str(msg))
                navailable -= 1
                nwaiting += 1
            failed = False
            while not failed and nwaiting > 0:
                # TODO adjust timeout value
                result = poller.poll(25)
                if self.socket_receive in result:
                    answer = Message.from_string(self.socket_receive.recv_string())
                    logging.debug('Received {}'.format(str(answer)))
                    if answer.type == 'RESULT':
                        if best_value is None or best_value > answer.content:
                            best_value = answer.content
                            best_solution = solution
                        nwaiting -= 1
                        trecv += 1
                        navailable += 1
                    if answer.type == 'STOP':
                        logging.error('Strategy: stopping.')
                        # TODO fix this,
                else:
                    failed = True
        logging.debug('Strategy: best found {}'.format(best_value))
        logging.debug('Strategy: best solution {}'.format(best_solution))
        self.socket_send.send_string(str(self.messenger.build_stop_message()))
        msg = self.messenger.build_result_message(content={
            'value': best_value,
            'solution': best_solution
        })
        logging.debug('Strategy: sending result {}'.format(str(msg)))
        self.socket_send.send_string(str(msg))
        poller.unregister(self.socket_receive)

if __name__ == '__main__':
    import sys
    ZDTStrategy(*sys.argv[1:]).start()
<commit_msg>Remove distributed mode for ZDT, it was incomplete.<commit_after>import random
import logging

from zephyrus.message import Message
from zephyrus.strategy import Strategy

class ZDTStrategy(Strategy):
    def configure(self, content):
        self.niter = content['niter']
        self.length = content['length']

    def mainloop(self):
        # TODO expecting configuration here?
        best_solution = None
        best_value = None
        for i in range(self.niter):
            logging.info("Strategy: progress {}/{}".format(i + 1, self.niter))
            solution = [random.random() for _ in range(self.length)]
            msg = self.messenger.build_evaluate_message(content={'id': None, 'data': solution})
            self.socket_send.send_string(str(msg))
            ans = Message.from_string(self.socket_receive.recv_string())
            logging.debug('Strategy: received {}'.format(str(ans)))
            value = ans.content['data']
            if ans.type == 'RESULT':
                if best_value is None or best_value > value:
                    best_value = value
                    best_solution = solution
            elif ans.type == 'STOP':
                logging.info('Strategy: stopping.')
                break
        logging.debug('Strategy: best found {}'.format(best_value))
        logging.debug('Strategy: best solution {}'.format(best_solution))
        self.socket_send.send_string(str(self.messenger.build_stop_message()))
        msg = self.messenger.build_result_message(content={
            'value': best_value,
            'solution': best_solution
        })
        logging.debug('Strategy: sending result {}'.format(str(msg)))
        self.socket_send.send_string(str(msg))

if __name__ == '__main__':
    import sys
    ZDTStrategy(*sys.argv[1:]).start()
<|endoftext|>


<commit_before>import re
import requests

from django.conf import settings
from django.utils.encoding import smart_text
import magic
from typing import Any, Optional, Dict
from typing.re import Match

from version import ZULIP_VERSION
from zerver.lib.cache import cache_with_key, get_cache_with_key, preview_url_cache_key
from zerver.lib.url_preview.oembed import get_oembed_data
from zerver.lib.url_preview.parsers import OpenGraphParser, GenericParser

# FIXME: Should we use a database cache or a memcached in production? What if
# opengraph data is changed for a site?
# Use an in-memory cache for development, to make it easy to develop this code
CACHE_NAME = "database" if not settings.DEVELOPMENT else "in-memory"
# Based on django.core.validators.URLValidator, with ftp support removed.
link_regex = re.compile(
    r'^(?:http)s?://'  # http:// or https://
    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
    r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # ...or ip
    r'(?::\d+)?'  # optional port
    r'(?:/?|[/?]\S+)$', re.IGNORECASE)
# FIXME: This header is not used by pyoembed, when trying to autodiscover!
# Set a custom user agent, since some sites block us with the default requests header
HEADERS = {'User-Agent': 'Zulip URL preview/%s' % (ZULIP_VERSION,)}

def is_link(url: str) -> Match[str]:
    return link_regex.match(smart_text(url))

def guess_mimetype_from_content(response: requests.Response) -> str:
    mime_magic = magic.Magic(mime=True)
    try:
        content = next(response.iter_content(1000))
    except StopIteration:
        content = ''
    return mime_magic.from_buffer(content)

def valid_content_type(url: str) -> bool:
    try:
        response = requests.get(url, stream=True, headers=HEADERS)
    except requests.RequestException:
        return False

    if not response.ok:
        return False

    content_type = response.headers.get('content-type')
    # Be accommodating of bad servers: assume content may be html if no content-type header
    if not content_type or content_type.startswith('text/html'):
        # Verify that the content is actually HTML if the server claims it is
        content_type = guess_mimetype_from_content(response)
    return content_type.startswith('text/html')

@cache_with_key(preview_url_cache_key, cache_name=CACHE_NAME, with_statsd_key="urlpreview_data")
def get_link_embed_data(url: str,
                        maxwidth: Optional[int]=640,
                        maxheight: Optional[int]=480) -> Optional[Dict[str, Any]]:
    if not is_link(url):
        return None

    if not valid_content_type(url):
        return None

    # Fetch information from URL.
    # We are using three sources in next order:
    # 1. OEmbed
    # 2. Open Graph
    # 3. Meta tags
    try:
        data = get_oembed_data(url, maxwidth=maxwidth, maxheight=maxheight)
    except requests.exceptions.RequestException:
        # This is what happens if the target URL cannot be fetched; in
        # that case, there's nothing to do here, and this URL has no
        # open graph data.
        return None
    data = data or {}
    response = requests.get(url, stream=True, headers=HEADERS)

    if response.ok:
        og_data = OpenGraphParser(response.text).extract_data()
        if og_data:
            data.update(og_data)
        generic_data = GenericParser(response.text).extract_data() or {}
        for key in ['title', 'description', 'image']:
            if not data.get(key) and generic_data.get(key):
                data[key] = generic_data[key]
    return data

@get_cache_with_key(preview_url_cache_key, cache_name=CACHE_NAME)
def link_embed_data_from_cache(url: str, maxwidth: Optional[int]=640, maxheight: Optional[int]=480) -> Any:
    return
<commit_msg>url preview: Timeout requests after 15 seconds.<commit_after>import re
import requests

from django.conf import settings
from django.utils.encoding import smart_text
import magic
from typing import Any, Optional, Dict
from typing.re import Match

from version import ZULIP_VERSION
from zerver.lib.cache import cache_with_key, get_cache_with_key, preview_url_cache_key
from zerver.lib.url_preview.oembed import get_oembed_data
from zerver.lib.url_preview.parsers import OpenGraphParser, GenericParser

# FIXME: Should we use a database cache or a memcached in production? What if
# opengraph data is changed for a site?
# Use an in-memory cache for development, to make it easy to develop this code
CACHE_NAME = "database" if not settings.DEVELOPMENT else "in-memory"
# Based on django.core.validators.URLValidator, with ftp support removed.
link_regex = re.compile(
    r'^(?:http)s?://'  # http:// or https://
    r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|'  # domain...
    r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'  # ...or ip
    r'(?::\d+)?'  # optional port
    r'(?:/?|[/?]\S+)$', re.IGNORECASE)
# FIXME: This header and timeout are not used by pyoembed, when trying to autodiscover!
# Set a custom user agent, since some sites block us with the default requests header
HEADERS = {'User-Agent': 'Zulip URL preview/%s' % (ZULIP_VERSION,)}
TIMEOUT = 15

def is_link(url: str) -> Match[str]:
    return link_regex.match(smart_text(url))

def guess_mimetype_from_content(response: requests.Response) -> str:
    mime_magic = magic.Magic(mime=True)
    try:
        content = next(response.iter_content(1000))
    except StopIteration:
        content = ''
    return mime_magic.from_buffer(content)

def valid_content_type(url: str) -> bool:
    try:
        response = requests.get(url, stream=True, headers=HEADERS, timeout=TIMEOUT)
    except requests.RequestException:
        return False

    if not response.ok:
        return False

    content_type = response.headers.get('content-type')
    # Be accommodating of bad servers: assume content may be html if no content-type header
    if not content_type or content_type.startswith('text/html'):
        # Verify that the content is actually HTML if the server claims it is
        content_type = guess_mimetype_from_content(response)
    return content_type.startswith('text/html')

@cache_with_key(preview_url_cache_key, cache_name=CACHE_NAME, with_statsd_key="urlpreview_data")
def get_link_embed_data(url: str,
                        maxwidth: Optional[int]=640,
                        maxheight: Optional[int]=480) -> Optional[Dict[str, Any]]:
    if not is_link(url):
        return None

    if not valid_content_type(url):
        return None

    # Fetch information from URL.
    # We are using three sources in next order:
    # 1. OEmbed
    # 2. Open Graph
    # 3. Meta tags
    try:
        data = get_oembed_data(url, maxwidth=maxwidth, maxheight=maxheight)
    except requests.exceptions.RequestException:
        # This is what happens if the target URL cannot be fetched; in
        # that case, there's nothing to do here, and this URL has no
        # open graph data.
        return None
    data = data or {}
    response = requests.get(url, stream=True, headers=HEADERS, timeout=TIMEOUT)

    if response.ok:
        og_data = OpenGraphParser(response.text).extract_data()
        if og_data:
            data.update(og_data)
        generic_data = GenericParser(response.text).extract_data() or {}
        for key in ['title', 'description', 'image']:
            if not data.get(key) and generic_data.get(key):
                data[key] = generic_data[key]
    return data

@get_cache_with_key(preview_url_cache_key, cache_name=CACHE_NAME)
def link_embed_data_from_cache(url: str, maxwidth: Optional[int]=640, maxheight: Optional[int]=480) -> Any:
    return
<|endoftext|>


<commit_before># Copyright 2017 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Handler for the root of the sdk API."""

import datetime
import functools
import jwt
import os
import threading

from zvmsdk import config
from zvmsdk import exception
from zvmsdk import log
from zvmsdk.sdkwsgi import util

CONF = config.CONF
LOG = log.LOG

DEFAULT_TOKEN_VALIDATION_PERIOD = 3600
TOKEN_LOCK = threading.Lock()

def get_admin_token(path):
    if os.path.exists(path):
        TOKEN_LOCK.acquire()
        try:
            with open(path, 'r') as fd:
                token = fd.read().strip()
        except Exception:
            LOG.debug('token file open failed.')
            raise exception.ZVMUnauthorized()
        finally:
            TOKEN_LOCK.release()
    else:
        LOG.debug('token configuration file not found.')
        raise exception.ZVMUnauthorized()
    return token

@util.SdkWsgify
def create(req):
    if 'X-Admin-Token' not in req.headers:
        LOG.debug('no X-Admin-Token given in reqeust header')
        raise exception.ZVMUnauthorized()
    token_file_path = CONF.wsgi.token_path
    admin_token = get_admin_token(token_file_path)
    if (req.headers['X-Admin-Token'] != admin_token):
        LOG.debug('X-Admin-Token incorrect')
        raise exception.ZVMUnauthorized()

    expires = CONF.wsgi.token_validation_period
    if expires < 0:
        expires = DEFAULT_TOKEN_VALIDATION_PERIOD

    expired_elapse = datetime.timedelta(seconds=expires)
    expired_time = datetime.datetime.utcnow() + expired_elapse
    payload = {'exp': expired_time}
    user_token = jwt.encode(payload, admin_token)

    req.response.headers.add('X-Auth-Token', user_token)

    return req.response

# To validate the token, it is possible the token is expired or the
# token is not validated at all
def validate(function):
    @functools.wraps(function)
    def wrap_func(req, *args, **kwargs):

        # by default, no token validation used
        if CONF.wsgi.auth == 'none':
            return function(req, *args, **kwargs)

        # so, this is for token validation
        if 'X-Auth-Token' not in req.headers:
            LOG.debug('no X-Auth-Token given in reqeust header')
            raise exception.ZVMUnauthorized()

        token_file_path = CONF.wsgi.token_path
        admin_token = get_admin_token(token_file_path)
        try:
            jwt.decode(req.headers['X-Auth-Token'], admin_token)
        except jwt.ExpiredSignatureError:
            LOG.debug('token validation failed because it is expired')
            raise exception.ZVMUnauthorized()
        except jwt.DecodeError:
            LOG.debug('token not valid')
            raise exception.ZVMUnauthorized()
        except Exception:
            LOG.debug('unknown exception occur during token validation')
            raise exception.ZVMUnauthorized()

        return function(req, *args, **kwargs)
    return wrap_func
<commit_msg>fix bug in token server side<commit_after># Copyright 2017 IBM Corp.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""Handler for the root of the sdk API."""

import datetime
import functools
import jwt
import os
import threading

from zvmsdk import config
from zvmsdk import exception
from zvmsdk import log
from zvmsdk.sdkwsgi import util

CONF = config.CONF
LOG = log.LOG

DEFAULT_TOKEN_VALIDATION_PERIOD = 3600
TOKEN_LOCK = threading.Lock()

def get_admin_token(path):
    if os.path.exists(path):
        TOKEN_LOCK.acquire()
        try:
            with open(path, 'r') as fd:
                token = fd.read().strip()
        except Exception:
            LOG.debug('token file open failed.')
            raise exception.ZVMUnauthorized()
        finally:
            TOKEN_LOCK.release()
    else:
        LOG.debug('token configuration file not found.')
        raise exception.ZVMUnauthorized()
    return token

@util.SdkWsgify
def create(req):
    # Check if token validation closed
    if CONF.wsgi.auth.lower() == 'none':
        user_token = 'server-auth-closed'
        req.response.headers.add('X-Auth-Token', user_token)
        return req.response
    # Validation is open, so start to validate the admin-token
    if 'X-Admin-Token' not in req.headers:
        LOG.debug('no X-Admin-Token given in reqeust header')
        raise exception.ZVMUnauthorized()
    token_file_path = CONF.wsgi.token_path
    admin_token = get_admin_token(token_file_path)
    if (req.headers['X-Admin-Token'] != admin_token):
        LOG.debug('X-Admin-Token incorrect')
        raise exception.ZVMUnauthorized()

    expires = CONF.wsgi.token_validation_period
    if expires < 0:
        expires = DEFAULT_TOKEN_VALIDATION_PERIOD

    expired_elapse = datetime.timedelta(seconds=expires)
    expired_time = datetime.datetime.utcnow() + expired_elapse
    payload = {'exp': expired_time}
    user_token = jwt.encode(payload, admin_token)

    req.response.headers.add('X-Auth-Token', user_token)

    return req.response

# To validate the token, it is possible the token is expired or the
# token is not validated at all
def validate(function):
    @functools.wraps(function)
    def wrap_func(req, *args, **kwargs):

        # by default, no token validation used
        if CONF.wsgi.auth.lower() == 'none':
            return function(req, *args, **kwargs)

        # so, this is for token validation
        if 'X-Auth-Token' not in req.headers:
            LOG.debug('no X-Auth-Token given in reqeust header')
            raise exception.ZVMUnauthorized()

        token_file_path = CONF.wsgi.token_path
        admin_token = get_admin_token(token_file_path)
        try:
            jwt.decode(req.headers['X-Auth-Token'], admin_token)
        except jwt.ExpiredSignatureError:
            LOG.debug('token validation failed because it is expired')
            raise exception.ZVMUnauthorized()
        except jwt.DecodeError:
            LOG.debug('token not valid')
            raise exception.ZVMUnauthorized()
        except Exception:
            LOG.debug('unknown exception occur during token validation')
            raise exception.ZVMUnauthorized()

        return function(req, *args, **kwargs)
    return wrap_func
<|endoftext|>


<commit_before>import socket
import time
import httplib
from urllib import urlencode
from threading import Lock, Event
from django.conf import settings
from django.core.cache import cache
from graphite.node import LeafNode, BranchNode
from graphite.intervals import Interval, IntervalSet
from graphite.readers import FetchInProgress
from graphite.logger import log

try:
  import cPickle as pickle
except ImportError:
  import pickle

class RemoteStore(object):
  lastFailure = 0.0
  available = property(lambda self: time.time() - self.lastFailure > settings.REMOTE_RETRY_DELAY)

  def __init__(self, host):
    self.host = host

  def find(self, query):
    request = FindRequest(self, query)
    request.send()
    return request

  def fail(self):
    self.lastFailure = time.time()

class FindRequest(object):
  __slots__ = ('store', 'query', 'connection',
               'failed', 'cacheKey', 'cachedResult')

  def __init__(self, store, query):
    self.store = store
    self.query = query
    self.connection = None
    self.failed = False

    if query.startTime:
      start = query.startTime - (query.startTime % settings.FIND_CACHE_DURATION)
    else:
      start = ""

    if query.endTime:
      end = query.endTime - (query.endTime % settings.FIND_CACHE_DURATION)
    else:
      end = ""

    self.cacheKey = "find:%s:%s:%s:%s" % (store.host, query.pattern, start, end)
    self.cachedResult = None

  def send(self):
    log.info("FindRequest.send(host=%s, query=%s) called" % (self.store.host, self.query))

    self.cachedResult = cache.get(self.cacheKey)
    if self.cachedResult is not None:
      log.info("FindRequest(host=%s, query=%s) using cached result" % (self.store.host, self.query))
      return

    self.connection = HTTPConnectionWithTimeout(self.store.host)
    self.connection.timeout = settings.REMOTE_FIND_TIMEOUT

    query_params = [
      ('local', '1'),
      ('format', 'pickle'),
      ('query', self.query.pattern),
    ]
    if self.query.startTime:
      query_params.append( ('from', self.query.startTime) )

    if self.query.endTime:
      query_params.append( ('until', self.query.endTime) )

    query_string = urlencode(query_params)

    try:
      self.connection.request('GET', '/metrics/find/?' + query_string)
    except:
      log.exception("FindRequest.send(host=%s, query=%s) exception during request" % (self.store.host, self.query))
      self.store.fail()
      self.failed = True

  def get_results(self):
    if self.failed:
      return

    if self.cachedResult is not None:
      results = self.cachedResult
    else:
      if self.connection is None:
        self.send()

      try:
        response = self.connection.getresponse()
        assert response.status == 200, "received error response %s - %s" % (response.status, response.reason)
        result_data = response.read()
        results = pickle.loads(result_data)

      except:
        log.exception("FindRequest.get_results(host=%s, query=%s) exception processing response" % (self.store.host, self.query))
        self.store.fail()
        return

      cache.set(self.cacheKey, results, settings.FIND_CACHE_DURATION)

    for node_info in results:
      if node_info.get('is_leaf'):
        reader = RemoteReader(self.store, node_info, bulk_query=self.query.pattern)
        node = LeafNode(node_info['path'], reader)
      else:
        node = BranchNode(node_info['path'])

      node.local = False
      yield node

class RemoteReader(object):
  __slots__ = ('store', 'metric_path', 'intervals', 'query')
  cache_lock = Lock()
  request_cache = {}
  request_locks = {}
  request_times = {}

  def __init__(self, store, node_info, bulk_query=None):
    self.store = store
    self.metric_path = node_info['path']
    self.intervals = node_info['intervals']
    self.query = bulk_query or node_info['path']

  def __repr__(self):
    return '<RemoteReader[%x]: %s>' % (id(self), self.store.host)

  def get_intervals(self):
    return self.intervals

  def fetch(self, startTime, endTime):
    query_params = [
      ('target', self.query),
      ('format', 'pickle'),
      ('local', '1'),
      ('noCache', '1'),
      ('from', str( int(startTime) )),
      ('until', str( int(endTime) ))
    ]
    query_string = urlencode(query_params)
    urlpath = '/render/?' + query_string
    url = "http://%s%s" % (self.store.host, urlpath)

    # Quick cache check up front
    self.clean_cache()
    cached_results = self.request_cache.get(url)
    if cached_results:
      for series in cached_results:
        if series['name'] == self.metric_path:
          time_info = (series['start'], series['end'], series['step'])
          return (time_info, series['values'])

    # Synchronize with other RemoteReaders using the same bulk query.
    # Despite our use of thread synchronization primitives, the common
    # case is for synchronizing asynchronous fetch operations within
    # a single thread.
    (request_lock, wait_lock, completion_event) = self.get_request_locks(url)

    if request_lock.acquire(False): # we only send the request the first time we're called
      try:
        log.info("RemoteReader.request_data :: requesting %s" % url)
        connection = HTTPConnectionWithTimeout(self.store.host)
        connection.timeout = settings.REMOTE_FETCH_TIMEOUT
        connection.request('GET', urlpath)
      except:
        completion_event.set()
        self.store.fail()
        log.exception("Error requesting %s" % url)
        raise

    def wait_for_results():
      if wait_lock.acquire(False): # the FetchInProgress that gets waited on waits for the actual completion
        try:
          response = connection.getresponse()
          if response.status != 200:
            raise Exception("Error response %d %s from %s" % (response.status, response.reason, url))

          pickled_response = response.read()
          results = pickle.loads(pickled_response)
          self.cache_lock.acquire()
          self.request_cache[url] = results
          self.cache_lock.release()
          completion_event.set()
          return results
        except:
          completion_event.set()
          self.store.fail()
          log.exception("Error requesting %s" % url)
          raise

      else: # otherwise we just wait on the completion_event
        completion_event.wait(settings.REMOTE_FETCH_TIMEOUT)
        cached_results = self.request_cache.get(url)
        if cached_results is None:
          raise Exception("Passive remote fetch failed to find cached results")
        else:
          return cached_results

    def extract_my_results():
      for series in wait_for_results():
        if series['name'] == self.metric_path:
          time_info = (series['start'], series['end'], series['step'])
          return (time_info, series['values'])

    return FetchInProgress(extract_my_results)

  def clean_cache(self):
    self.cache_lock.acquire()
    try:
      if len(self.request_locks) >= settings.REMOTE_READER_CACHE_SIZE_LIMIT:
        log.info("RemoteReader.request_data :: clearing old from request_cache and request_locks")
        now = time.time()
        for url, timestamp in self.request_times.items():
          age = now - timestamp
          if age >= (2 * settings.REMOTE_FETCH_TIMEOUT):
            del self.request_locks[url]
            del self.request_times[url]
            if url in self.request_cache:
              del self.request_cache[url]
    finally:
      self.cache_lock.release()

  def get_request_locks(self, url):
    self.cache_lock.acquire()
    try:
      if url not in self.request_locks:
        self.request_locks[url] = (Lock(), Lock(), Event())
        self.request_times[url] = time.time()
      return self.request_locks[url]
    finally:
      self.cache_lock.release()

# This is a hack to put a timeout in the connect() of an HTTP request.
# Python 2.6 supports this already, but many Graphite installations
# are not on 2.6 yet.

class HTTPConnectionWithTimeout(httplib.HTTPConnection):
  timeout = 30

  def connect(self):
    msg = "getaddrinfo returns an empty list"
    for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM):
      af, socktype, proto, canonname, sa = res
      try:
        self.sock = socket.socket(af, socktype, proto)
        try:
          self.sock.settimeout( float(self.timeout) ) # default self.timeout is an object() in 2.6
        except:
          pass
        self.sock.connect(sa)
        self.sock.settimeout(None)
      except socket.error, msg:
        if self.sock:
          self.sock.close()
          self.sock = None
          continue
      break
    if not self.sock:
      raise socket.error, msg
<commit_msg>Add the URL to the exception when remote failed to respond.<commit_after>import socket
import time
import httplib
from urllib import urlencode
from threading import Lock, Event
from django.conf import settings
from django.core.cache import cache
from graphite.node import LeafNode, BranchNode
from graphite.intervals import Interval, IntervalSet
from graphite.readers import FetchInProgress
from graphite.logger import log

try:
  import cPickle as pickle
except ImportError:
  import pickle

class RemoteStore(object):
  lastFailure = 0.0
  available = property(lambda self: time.time() - self.lastFailure > settings.REMOTE_RETRY_DELAY)

  def __init__(self, host):
    self.host = host

  def find(self, query):
    request = FindRequest(self, query)
    request.send()
    return request

  def fail(self):
    self.lastFailure = time.time()

class FindRequest(object):
  __slots__ = ('store', 'query', 'connection',
               'failed', 'cacheKey', 'cachedResult')

  def __init__(self, store, query):
    self.store = store
    self.query = query
    self.connection = None
    self.failed = False

    if query.startTime:
      start = query.startTime - (query.startTime % settings.FIND_CACHE_DURATION)
    else:
      start = ""

    if query.endTime:
      end = query.endTime - (query.endTime % settings.FIND_CACHE_DURATION)
    else:
      end = ""

    self.cacheKey = "find:%s:%s:%s:%s" % (store.host, query.pattern, start, end)
    self.cachedResult = None

  def send(self):
    log.info("FindRequest.send(host=%s, query=%s) called" % (self.store.host, self.query))

    self.cachedResult = cache.get(self.cacheKey)
    if self.cachedResult is not None:
      log.info("FindRequest(host=%s, query=%s) using cached result" % (self.store.host, self.query))
      return

    self.connection = HTTPConnectionWithTimeout(self.store.host)
    self.connection.timeout = settings.REMOTE_FIND_TIMEOUT

    query_params = [
      ('local', '1'),
      ('format', 'pickle'),
      ('query', self.query.pattern),
    ]
    if self.query.startTime:
      query_params.append( ('from', self.query.startTime) )

    if self.query.endTime:
      query_params.append( ('until', self.query.endTime) )

    query_string = urlencode(query_params)

    try:
      self.connection.request('GET', '/metrics/find/?' + query_string)
    except:
      log.exception("FindRequest.send(host=%s, query=%s) exception during request" % (self.store.host, self.query))
      self.store.fail()
      self.failed = True

  def get_results(self):
    if self.failed:
      return

    if self.cachedResult is not None:
      results = self.cachedResult
    else:
      if self.connection is None:
        self.send()

      try:
        response = self.connection.getresponse()
        assert response.status == 200, "received error response %s - %s" % (response.status, response.reason)
        result_data = response.read()
        results = pickle.loads(result_data)

      except:
        log.exception("FindRequest.get_results(host=%s, query=%s) exception processing response" % (self.store.host, self.query))
        self.store.fail()
        return

      cache.set(self.cacheKey, results, settings.FIND_CACHE_DURATION)

    for node_info in results:
      if node_info.get('is_leaf'):
        reader = RemoteReader(self.store, node_info, bulk_query=self.query.pattern)
        node = LeafNode(node_info['path'], reader)
      else:
        node = BranchNode(node_info['path'])

      node.local = False
      yield node

class RemoteReader(object):
  __slots__ = ('store', 'metric_path', 'intervals', 'query')
  cache_lock = Lock()
  request_cache = {}
  request_locks = {}
  request_times = {}

  def __init__(self, store, node_info, bulk_query=None):
    self.store = store
    self.metric_path = node_info['path']
    self.intervals = node_info['intervals']
    self.query = bulk_query or node_info['path']

  def __repr__(self):
    return '<RemoteReader[%x]: %s>' % (id(self), self.store.host)

  def get_intervals(self):
    return self.intervals

  def fetch(self, startTime, endTime):
    query_params = [
      ('target', self.query),
      ('format', 'pickle'),
      ('local', '1'),
      ('noCache', '1'),
      ('from', str( int(startTime) )),
      ('until', str( int(endTime) ))
    ]
    query_string = urlencode(query_params)
    urlpath = '/render/?' + query_string
    url = "http://%s%s" % (self.store.host, urlpath)

    # Quick cache check up front
    self.clean_cache()
    cached_results = self.request_cache.get(url)
    if cached_results:
      for series in cached_results:
        if series['name'] == self.metric_path:
          time_info = (series['start'], series['end'], series['step'])
          return (time_info, series['values'])

    # Synchronize with other RemoteReaders using the same bulk query.
    # Despite our use of thread synchronization primitives, the common
    # case is for synchronizing asynchronous fetch operations within
    # a single thread.
    (request_lock, wait_lock, completion_event) = self.get_request_locks(url)

    if request_lock.acquire(False): # we only send the request the first time we're called
      try:
        log.info("RemoteReader.request_data :: requesting %s" % url)
        connection = HTTPConnectionWithTimeout(self.store.host)
        connection.timeout = settings.REMOTE_FETCH_TIMEOUT
        connection.request('GET', urlpath)
      except:
        completion_event.set()
        self.store.fail()
        log.exception("Error requesting %s" % url)
        raise

    def wait_for_results():
      if wait_lock.acquire(False): # the FetchInProgress that gets waited on waits for the actual completion
        try:
          response = connection.getresponse()
          if response.status != 200:
            raise Exception("Error response %d %s from %s" % (response.status, response.reason, url))

          pickled_response = response.read()
          results = pickle.loads(pickled_response)
          self.cache_lock.acquire()
          self.request_cache[url] = results
          self.cache_lock.release()
          completion_event.set()
          return results
        except:
          completion_event.set()
          self.store.fail()
          log.exception("Error requesting %s" % url)
          raise

      else: # otherwise we just wait on the completion_event
        completion_event.wait(settings.REMOTE_FETCH_TIMEOUT)
        cached_results = self.request_cache.get(url)
        if cached_results is None:
          raise Exception("Passive remote fetch failed to find cached results for %s" % url)
        else:
          return cached_results

    def extract_my_results():
      for series in wait_for_results():
        if series['name'] == self.metric_path:
          time_info = (series['start'], series['end'], series['step'])
          return (time_info, series['values'])

    return FetchInProgress(extract_my_results)

  def clean_cache(self):
    self.cache_lock.acquire()
    try:
      if len(self.request_locks) >= settings.REMOTE_READER_CACHE_SIZE_LIMIT:
        log.info("RemoteReader.request_data :: clearing old from request_cache and request_locks")
        now = time.time()
        for url, timestamp in self.request_times.items():
          age = now - timestamp
          if age >= (2 * settings.REMOTE_FETCH_TIMEOUT):
            del self.request_locks[url]
            del self.request_times[url]
            if url in self.request_cache:
              del self.request_cache[url]
    finally:
      self.cache_lock.release()

  def get_request_locks(self, url):
    self.cache_lock.acquire()
    try:
      if url not in self.request_locks:
        self.request_locks[url] = (Lock(), Lock(), Event())
        self.request_times[url] = time.time()
      return self.request_locks[url]
    finally:
      self.cache_lock.release()

# This is a hack to put a timeout in the connect() of an HTTP request.
# Python 2.6 supports this already, but many Graphite installations
# are not on 2.6 yet.

class HTTPConnectionWithTimeout(httplib.HTTPConnection):
  timeout = 30

  def connect(self):
    msg = "getaddrinfo returns an empty list"
    for res in socket.getaddrinfo(self.host, self.port, 0, socket.SOCK_STREAM):
      af, socktype, proto, canonname, sa = res
      try:
        self.sock = socket.socket(af, socktype, proto)
        try:
          self.sock.settimeout( float(self.timeout) ) # default self.timeout is an object() in 2.6
        except:
          pass
        self.sock.connect(sa)
        self.sock.settimeout(None)
      except socket.error, msg:
        if self.sock:
          self.sock.close()
          self.sock = None
          continue
      break
    if not self.sock:
      raise socket.error, msg
<|endoftext|>


<commit_before>"""
Scraper for Florida 3rd District Court of Appeal
CourtID: flaapp3
Court Short Name: flaapp3

History:
 - 2014-07-21: Written by Andrei Chelaru
 - 2014-07-24: Reviewed by mlr
 - 2015-07-28: Updated by m4h7
"""
import re
import time
from lxml import html
from datetime import date

from juriscraper.lib.string_utils import titlecase
from juriscraper.OpinionSite import OpinionSite

class Site(OpinionSite):
    def __init__(self, *args, **kwargs):
        super(Site, self).__init__(*args, **kwargs)
        self.court_id = self.__module__
        self.year = date.today().year
        self.url = 'http://www.3dca.flcourts.org/Opinions/ArchivedOpinions.shtml'
        self.base_path = "//h3/following::text()[.='OPINIONS']/following::table[1]//tr"

    def _download(self, request_dict={}):
        html_l = super(Site, self)._download(request_dict)
        html_trees = []
        # this path reads the row for the last month in that year
        path = "//th[contains(., '{year}')]/following::tr[1]/td[position()>1]/a[contains(., '/')]/@href".format(
            year=self.year
        )
        # to get all the dates in that page the following path can be used:
        # path = "//th/following::tr/td[position()>1]/a[contains(., '/')]/@href"
        for url in html_l.xpath(path):
            html_tree = self._get_html_tree_by_url(url, request_dict)
            html_trees.append(html_tree)
        return html_trees

    def _get_case_names(self):
        case_names = []
        for html_tree in self.html:
            case_names.extend(self._return_case_names(html_tree))
        return case_names

    def _return_case_names(self, html_tree):
        path = "{base}/td[2]//text()[1]".format(base=self.base_path)
        return [titlecase(s.lower()) for s in html_tree.xpath(path)]

    def _get_download_urls(self):
        download_urls = []
        for html_tree in self.html:
            download_urls.extend(self._return_download_urls(html_tree))
        return download_urls

    def _return_download_urls(self, html_tree):
        path = "{base}/td[1]//a/@href".format(base=self.base_path)
        return list(html_tree.xpath(path))

    def _get_case_dates(self):
        case_dates = []
        for html_tree in self.html:
            case_dates.extend(self._return_dates(html_tree))
        return case_dates

    def _return_dates(self, html_tree):
        path = "//h3/text()"
        dates = []
        text = html_tree.xpath(path)[0]
        text = re.search('(\d{2}-\d{2}-\d{4})', text).group(1)
        case_date = date.fromtimestamp(time.mktime(time.strptime(text.strip(), '%m-%d-%Y')))
        dates.extend([case_date] * int(html_tree.xpath("count({base})".format(base=self.base_path))))
        return dates

    def _get_precedential_statuses(self):
        return ['Published'] * len(self.case_dates)

    def _get_docket_numbers(self):
        docket_numbers = []
        for html_tree in self.html:
            docket_numbers.extend(self._return_docket_numbers(html_tree))
        return docket_numbers

    def _return_docket_numbers(self, html_tree):
        docket_numbers = []
        for e in html_tree.xpath("{base}/td[1]//a".format(base=self.base_path)):
            s = html.tostring(e, method='text', encoding='unicode')
            docket_numbers.append(s)
        return docket_numbers
<commit_msg>Adding contact info to fladistctapp_3<commit_after>"""
Scraper for Florida 3rd District Court of Appeal
CourtID: flaapp3
Court Short Name: flaapp3
Contact: 3dca@flcourts.org, Angel Falero <faleroa@flcourts.org> (305-229-6743)

History:
 - 2014-07-21: Written by Andrei Chelaru
 - 2014-07-24: Reviewed by mlr
 - 2015-07-28: Updated by m4h7
"""
import re
import time
from lxml import html
from datetime import date

from juriscraper.lib.string_utils import titlecase
from juriscraper.OpinionSite import OpinionSite

class Site(OpinionSite):
    def __init__(self, *args, **kwargs):
        super(Site, self).__init__(*args, **kwargs)
        self.court_id = self.__module__
        self.year = date.today().year
        self.url = 'http://www.3dca.flcourts.org/Opinions/ArchivedOpinions.shtml'
        self.base_path = "//h3/following::text()[.='OPINIONS']/following::table[1]//tr"

    def _download(self, request_dict={}):
        html_l = super(Site, self)._download(request_dict)
        html_trees = []
        # this path reads the row for the last month in that year
        path = "//th[contains(., '{year}')]/following::tr[1]/td[position()>1]/a[contains(., '/')]/@href".format(
            year=self.year
        )
        # to get all the dates in that page the following path can be used:
        # path = "//th/following::tr/td[position()>1]/a[contains(., '/')]/@href"
        for url in html_l.xpath(path):
            html_tree = self._get_html_tree_by_url(url, request_dict)
            html_trees.append(html_tree)
        return html_trees

    def _get_case_names(self):
        case_names = []
        for html_tree in self.html:
            case_names.extend(self._return_case_names(html_tree))
        return case_names

    def _return_case_names(self, html_tree):
        path = "{base}/td[2]//text()[1]".format(base=self.base_path)
        return [titlecase(s.lower()) for s in html_tree.xpath(path)]

    def _get_download_urls(self):
        download_urls = []
        for html_tree in self.html:
            download_urls.extend(self._return_download_urls(html_tree))
        return download_urls

    def _return_download_urls(self, html_tree):
        path = "{base}/td[1]//a/@href".format(base=self.base_path)
        return list(html_tree.xpath(path))

    def _get_case_dates(self):
        case_dates = []
        for html_tree in self.html:
            case_dates.extend(self._return_dates(html_tree))
        return case_dates

    def _return_dates(self, html_tree):
        path = "//h3/text()"
        dates = []
        text = html_tree.xpath(path)[0]
        text = re.search('(\d{2}-\d{2}-\d{4})', text).group(1)
        case_date = date.fromtimestamp(time.mktime(time.strptime(text.strip(), '%m-%d-%Y')))
        dates.extend([case_date] * int(html_tree.xpath("count({base})".format(base=self.base_path))))
        return dates

    def _get_precedential_statuses(self):
        return ['Published'] * len(self.case_dates)

    def _get_docket_numbers(self):
        docket_numbers = []
        for html_tree in self.html:
            docket_numbers.extend(self._return_docket_numbers(html_tree))
        return docket_numbers

    def _return_docket_numbers(self, html_tree):
        docket_numbers = []
        for e in html_tree.xpath("{base}/td[1]//a".format(base=self.base_path)):
            s = html.tostring(e, method='text', encoding='unicode')
            docket_numbers.append(s)
        return docket_numbers
<|endoftext|>


<commit_before>##########################################################################
# Copyright (c) 2009, ETH Zurich.
# All rights reserved.
#
# This file is distributed under the terms in the attached LICENSE file.
# If you do not find this file, copies can be found by writing to:
# ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
##########################################################################

import tests
from common import TestCommon
from results import PassFailResult

@tests.add_test
class GrubBootTest(TestCommon):
    '''Simple test that checks if the machine boots into grub'''
    name = "grubboot"

    def setup(self, build, machine, testdir):
        # Don't build BF
        machine.lock()
        machine.setup()

    def run(self, build, machine, testdir):
        self.reboot(machine)
        # Ignore the boot phase
        self.boot_phase = False
        return self.collect_data(machine)

    def cleanup(self, machine):
        machine.shutdown()
        machine.unlock()

    def get_finish_string(self):
        return "GNU GRUB"

    def process_data(self, testdir, rawiter):
        for line in rawiter:
            if line.find("GNU GRUB  version 0.97-os.3") > -1:
                return PassFailResult(True)
        return PassFailResult(False)

<commit_msg>harness: also detect start of hagfish<commit_after>##########################################################################
# Copyright (c) 2009, ETH Zurich.
# All rights reserved.
#
# This file is distributed under the terms in the attached LICENSE file.
# If you do not find this file, copies can be found by writing to:
# ETH Zurich D-INFK, Haldeneggsteig 4, CH-8092 Zurich. Attn: Systems Group.
##########################################################################

import tests
from common import TestCommon
from results import PassFailResult

@tests.add_test
class GrubBootTest(TestCommon):
    '''Simple test that checks if the machine boots into grub'''
    name = "grubboot"
    grub_boot = "GNU GRUB  version 0.97-os.3"
    hagfish_boot = "Hagfish UEFI loader starting"

    def setup(self, build, machine, testdir):
        # Don't build BF
        machine.lock()
        machine.setup()

    def run(self, build, machine, testdir):
        self.reboot(machine)
        # Ignore the boot phase
        self.boot_phase = False
        return self.collect_data(machine)

    def cleanup(self, machine):
        machine.shutdown()
        machine.unlock()

    def is_finished(self, line):
        # Exit test when we get an assertion failure or an abort, rather than
        # waiting for timeout
        return self.grub_boot in line or \
               self.hagfish_boot in line or \
               line.startswith("Assertion failed on core") or \
               line.startswith("Aborted")

    def process_data(self, testdir, rawiter):
        for line in rawiter:
            if line.find(self.grub_boot) > -1 or \
               line.find(self.hagfish_boot) > -1:
                return PassFailResult(True)
        return PassFailResult(False)

<|endoftext|>


<commit_before># Always prefer setuptools over distutils
from setuptools import setup, find_packages

import os
import shutil
import sys

with open('README.rst', encoding='utf-8') as fobj:
    LONG_DESCRIPTION = fobj.read()

command = sys.argv[1]

if command == "sdist":
    # before creating the distribution, copy files from other locations in
    # the repository
    print("copying files...")
    this_dir = os.getcwd()
    root_dir = os.path.dirname(this_dir)
    src_dir = os.path.join(root_dir, "www", "src")
    data_dir = os.path.join(this_dir, "data")

    # copy files from /www/src
    for fname in ["brython.js", "brython_stdlib.js", "unicode.txt"]:
        shutil.copyfile(os.path.join(src_dir, fname),
            os.path.join(data_dir, fname))

    # copy demo.html
    with open(os.path.join(root_dir, 'www', 'demo.html'), encoding="utf-8") as f:
        demo = f.read()
    start_tag = "<!-- start copy -->"
    end_tag = "<!-- end copy -->"
    start = demo.find(start_tag)
    if start == -1:
        raise Exception("No tag <!-- start copy --> in demo.html")
    end = demo.find(end_tag)
    if end == -1:
        raise Exception("No tag <!-- end copy --> in demo.html")
    body = demo[start + len(start_tag) : end].strip()

    with open(os.path.join(data_dir, "demo.tmpl"), encoding="utf-8") as f:
        template = f.read()

    demo = template.replace("{{body}}", body)

    with open(os.path.join(data_dir, "demo.html"), "w", encoding="utf-8") as out:
        out.write(demo)

setup(
    name='brython',

    version='3.6.1',
    description='Brython is an implementation of Python 3 running in the browser',

    long_description = LONG_DESCRIPTION,

    # The project's main homepage.
    url='http://brython.info',

    # Author details
    author='Pierre Quentel',
    author_email='quentel.pierre@orange.fr',

    packages = ['data', 'data.tools'],

    # Choose your license
    license='BSD',

    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[

        # Indicate who your project is intended for
        'Intended Audience :: Developers',
        'Topic :: Software Development :: Interpreters',

        'Operating System :: OS Independent',

        # Pick your license as you wish (should match "license" above)
        'License :: OSI Approved :: BSD License',

        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 3',
    ],

    # What does your project relate to?
    keywords='Python browser',

    py_modules=["brython", "list_modules", "server"],

    package_data={
        'data': [
            'README.txt',
            'demo.html',
            'brython.js',
            'brython_stdlib.js',
            'unicode.txt'
            ]
    }

)<commit_msg>Change version to 3.6.2 in setup.py<commit_after># Always prefer setuptools over distutils
from setuptools import setup, find_packages

import os
import shutil
import sys

with open('README.rst', encoding='utf-8') as fobj:
    LONG_DESCRIPTION = fobj.read()

command = sys.argv[1]

if command == "sdist":
    # before creating the distribution, copy files from other locations in
    # the repository
    print("copying files...")
    this_dir = os.getcwd()
    root_dir = os.path.dirname(this_dir)
    src_dir = os.path.join(root_dir, "www", "src")
    data_dir = os.path.join(this_dir, "data")

    # copy files from /www/src
    for fname in ["brython.js", "brython_stdlib.js", "unicode.txt"]:
        shutil.copyfile(os.path.join(src_dir, fname),
            os.path.join(data_dir, fname))

    # copy demo.html
    with open(os.path.join(root_dir, 'www', 'demo.html'), encoding="utf-8") as f:
        demo = f.read()
    start_tag = "<!-- start copy -->"
    end_tag = "<!-- end copy -->"
    start = demo.find(start_tag)
    if start == -1:
        raise Exception("No tag <!-- start copy --> in demo.html")
    end = demo.find(end_tag)
    if end == -1:
        raise Exception("No tag <!-- end copy --> in demo.html")
    body = demo[start + len(start_tag) : end].strip()

    with open(os.path.join(data_dir, "demo.tmpl"), encoding="utf-8") as f:
        template = f.read()

    demo = template.replace("{{body}}", body)

    with open(os.path.join(data_dir, "demo.html"), "w", encoding="utf-8") as out:
        out.write(demo)

setup(
    name='brython',

    version='3.6.2',
    description='Brython is an implementation of Python 3 running in the browser',

    long_description = LONG_DESCRIPTION,

    # The project's main homepage.
    url='http://brython.info',

    # Author details
    author='Pierre Quentel',
    author_email='quentel.pierre@orange.fr',

    packages = ['data', 'data.tools'],

    # Choose your license
    license='BSD',

    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
    classifiers=[

        # Indicate who your project is intended for
        'Intended Audience :: Developers',
        'Topic :: Software Development :: Interpreters',

        'Operating System :: OS Independent',

        # Pick your license as you wish (should match "license" above)
        'License :: OSI Approved :: BSD License',

        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 3',
    ],

    # What does your project relate to?
    keywords='Python browser',

    py_modules=["brython", "list_modules", "server"],

    package_data={
        'data': [
            'README.txt',
            'demo.html',
            'brython.js',
            'brython_stdlib.js',
            'unicode.txt'
            ]
    }

)<|endoftext|>


<commit_before>#
# Copyright (c) 2016 Juniper Networks, Inc. All rights reserved.
#

"""
Kubernetes network manager
"""

import gevent
import random
from gevent.queue import Queue
from vnc.config_db import *
from vnc.reaction_map import REACTION_MAP

from vnc_api.vnc_api import *

from cfgm_common.vnc_amqp import VncAmqpHandle
from cfgm_common.zkclient import ZookeeperClient
import common.logger as logger
import common.args as kube_args
import vnc.vnc_kubernetes as vnc_kubernetes
import kube.kube_monitor as kube_monitor
import kube.namespace_monitor as namespace_monitor
import kube.pod_monitor as pod_monitor
import kube.service_monitor as service_monitor
import kube.network_policy_monitor as network_policy_monitor
import kube.endpoint_monitor as endpoint_monitor
import kube.ingress_monitor as ingress_monitor

class KubeNetworkManager(object):

    _kube_network_manager = None

    def __init__(self, args=None, logger=None, kube_api_connected=False, queue=None):
        if queue:
            self.q = queue
        else:
            self.q = Queue()
        self.args = args
        self.logger = logger
        # All monitors supported by this manager.
        self.monitors = {}
        self.kube = None
        while not kube_api_connected:
            try:
                self.kube = kube_monitor.KubeMonitor(
                    args=self.args, logger=self.logger, q=self.q)

                self.monitors['namespace'] = namespace_monitor.NamespaceMonitor(
                    args=self.args, logger=self.logger, q=self.q)

                self.monitors['pod'] = pod_monitor.PodMonitor(args=self.args,
                    logger=self.logger, q=self.q)

                self.monitors['service'] = service_monitor.ServiceMonitor(
                    args=self.args, logger=self.logger, q=self.q)

                self.monitors['network_policy'] =\
                    network_policy_monitor.NetworkPolicyMonitor(args=self.args,
                        logger=self.logger, q=self.q)

                self.monitors['endpoint'] = \
                    endpoint_monitor.EndPointMonitor(args=self.args,
                        logger=self.logger, q=self.q)

                self.monitors['ingress'] = \
                    ingress_monitor.IngressMonitor(args=self.args,
                        logger=self.logger, q=self.q)

                kube_api_connected = True

            except Exception as e:
                time.sleep(30)

        # Register all the known monitors.
        for monitor in self.monitors.values():
            monitor.register_monitor()

        self.vnc = vnc_kubernetes.VncKubernetes(args=self.args,
            logger=self.logger, q=self.q, kube=self.kube)
    # end __init__

    def _kube_object_cache_enabled(self):
        return True if self.args.kube_object_cache == 'True' else False;

    def launch_timer(self):
        if not self.args.kube_timer_interval.isdigit():
            self.logger.emergency("set seconds for kube_timer_interval "
                                  "in contrail-kubernetes.conf. \
                                   example: kube_timer_interval=60")
            sys.exit()
        self.logger.info("KubeNetworkManager - kube_timer_interval(%ss)"
                          %self.args.kube_timer_interval)
        time.sleep(int(self.args.kube_timer_interval))
        while True:
            gevent.sleep(int(self.args.kube_timer_interval))
            try:
                self.timer_callback()
            except Exception:
                pass

    def timer_callback(self):
        self.vnc.vnc_timer()

    def start_tasks(self):
        self.logger.info("Starting all tasks.")
        greenlets = [gevent.spawn(self.vnc.vnc_process)]
        for monitor in self.monitors.values():
            greenlets.append(gevent.spawn(monitor.event_callback))
        greenlets.append(gevent.spawn(self.launch_timer))
        gevent.joinall(greenlets)

    def reset(self):
        for cls in DBBaseKM.get_obj_type_map().values():
            cls.reset()

    @classmethod
    def get_instance(cls):
        return KubeNetworkManager._kube_network_manager

    @classmethod
    def destroy_instance(cls):
        inst = cls.get_instance()
        if inst is None:
            return
        inst.logger.sandesh_uninit()
        inst.vnc.destroy_instance()
        inst.vnc = None
        inst.q = None
        KubeNetworkManager._kube_network_manager = None

def run_kube_manager(km_logger, args, kube_api_skip, event_queue):
    kube_nw_mgr = KubeNetworkManager(args, km_logger, kube_api_connected=kube_api_skip, queue=event_queue)
    KubeNetworkManager._kube_network_manager = kube_nw_mgr
    kube_nw_mgr.start_tasks()

def main(args_str=None, kube_api_skip=False, event_queue=None):
    _zookeeper_client = None

    args = kube_args.parse_args(args_str)
    if 'kube_timer_interval' not in args:
        args.kube_timer_interval = '60'

    if args.cluster_id:
        client_pfx = args.cluster_id + '-'
        zk_path_pfx = args.cluster_id + '/'
    else:
        client_pfx = ''
        zk_path_pfx = ''

    # randomize collector list
    args.random_collectors = args.collectors
    if args.collectors:
        args.random_collectors = random.sample(args.collectors,
                                           len(args.collectors))

    km_logger = logger.KubeManagerLogger(args)

    if args.nested_mode == '0':
        # Initialize AMQP handler then close it to be sure remain queue of a
        # precedent run is cleaned
        try:
            vnc_amqp = VncAmqpHandle(km_logger, DBBaseKM,
                                     REACTION_MAP, 'kube_manager', args=args)
            vnc_amqp.establish()
            vnc_amqp.close()
        except Exception:
            pass
        finally:
            km_logger.debug("Removed remained AMQP queue")

        # Ensure zookeeper is up and running before starting kube-manager
        _zookeeper_client = ZookeeperClient(client_pfx+"kube-manager",
                                            args.zk_server_ip)

        km_logger.notice("Waiting to be elected as master...")
        _zookeeper_client.master_election(zk_path_pfx+"/kube-manager",
                                          os.getpid(), run_kube_manager,
                                          km_logger, args, kube_api_skip,
                                          event_queue)
        km_logger.notice("Elected master kube-manager node. Initializing...")

    else: #nested mode, skip zookeeper mastership check
        run_kube_manager(km_logger, args, kube_api_skip, event_queue)

if __name__ == '__main__':
    main()
<commit_msg>Kill all greenlet threads in KubeNetworkManager<commit_after>#
# Copyright (c) 2016 Juniper Networks, Inc. All rights reserved.
#

"""
Kubernetes network manager
"""

import gevent
import random
from gevent.queue import Queue
from vnc.config_db import *
from vnc.reaction_map import REACTION_MAP

from vnc_api.vnc_api import *

from cfgm_common.vnc_amqp import VncAmqpHandle
from cfgm_common.zkclient import ZookeeperClient
import common.logger as logger
import common.args as kube_args
import vnc.vnc_kubernetes as vnc_kubernetes
import kube.kube_monitor as kube_monitor
import kube.namespace_monitor as namespace_monitor
import kube.pod_monitor as pod_monitor
import kube.service_monitor as service_monitor
import kube.network_policy_monitor as network_policy_monitor
import kube.endpoint_monitor as endpoint_monitor
import kube.ingress_monitor as ingress_monitor

class KubeNetworkManager(object):

    _kube_network_manager = None

    def __init__(self, args=None, logger=None, kube_api_connected=False, queue=None):
        if queue:
            self.q = queue
        else:
            self.q = Queue()
        self.args = args
        self.logger = logger
        # All monitors supported by this manager.
        self.monitors = {}
        self.kube = None
        self.greenlets = []
        while not kube_api_connected:
            try:
                self.kube = kube_monitor.KubeMonitor(
                    args=self.args, logger=self.logger, q=self.q)

                self.monitors['namespace'] = namespace_monitor.NamespaceMonitor(
                    args=self.args, logger=self.logger, q=self.q)

                self.monitors['pod'] = pod_monitor.PodMonitor(args=self.args,
                    logger=self.logger, q=self.q)

                self.monitors['service'] = service_monitor.ServiceMonitor(
                    args=self.args, logger=self.logger, q=self.q)

                self.monitors['network_policy'] =\
                    network_policy_monitor.NetworkPolicyMonitor(args=self.args,
                        logger=self.logger, q=self.q)

                self.monitors['endpoint'] = \
                    endpoint_monitor.EndPointMonitor(args=self.args,
                        logger=self.logger, q=self.q)

                self.monitors['ingress'] = \
                    ingress_monitor.IngressMonitor(args=self.args,
                        logger=self.logger, q=self.q)

                kube_api_connected = True

            except Exception as e:
                time.sleep(30)

        # Register all the known monitors.
        for monitor in self.monitors.values():
            monitor.register_monitor()

        self.vnc = vnc_kubernetes.VncKubernetes(args=self.args,
            logger=self.logger, q=self.q, kube=self.kube)
    # end __init__

    def _kube_object_cache_enabled(self):
        return True if self.args.kube_object_cache == 'True' else False;

    def launch_timer(self):
        if not self.args.kube_timer_interval.isdigit():
            self.logger.emergency("set seconds for kube_timer_interval "
                                  "in contrail-kubernetes.conf. \
                                   example: kube_timer_interval=60")
            sys.exit()
        self.logger.info("KubeNetworkManager - kube_timer_interval(%ss)"
                          %self.args.kube_timer_interval)
        time.sleep(int(self.args.kube_timer_interval))
        while True:
            gevent.sleep(int(self.args.kube_timer_interval))
            try:
                self.timer_callback()
            except Exception:
                pass

    def timer_callback(self):
        self.vnc.vnc_timer()

    def start_tasks(self):
        self.logger.info("Starting all tasks.")
        self.greenlets = [gevent.spawn(self.vnc.vnc_process)]
        for monitor in self.monitors.values():
            self.greenlets.append(gevent.spawn(monitor.event_callback))
        self.greenlets.append(gevent.spawn(self.launch_timer))
        gevent.joinall(self.greenlets)

    def reset(self):
        for cls in DBBaseKM.get_obj_type_map().values():
            cls.reset()

    @classmethod
    def get_instance(cls):
        return KubeNetworkManager._kube_network_manager

    @classmethod
    def destroy_instance(cls):
        inst = cls.get_instance()
        if inst is None:
            return
        inst.logger.sandesh_uninit()
        for greenlet in inst.greenlets:
            greenlet.kill()
        inst.greenlets = []
        inst.vnc.destroy_instance()
        inst.vnc = None
        inst.q = None
        KubeNetworkManager._kube_network_manager = None

def run_kube_manager(km_logger, args, kube_api_skip, event_queue):
    kube_nw_mgr = KubeNetworkManager(args, km_logger, kube_api_connected=kube_api_skip, queue=event_queue)
    KubeNetworkManager._kube_network_manager = kube_nw_mgr
    kube_nw_mgr.start_tasks()

def main(args_str=None, kube_api_skip=False, event_queue=None):
    _zookeeper_client = None

    args = kube_args.parse_args(args_str)
    if 'kube_timer_interval' not in args:
        args.kube_timer_interval = '60'

    if args.cluster_id:
        client_pfx = args.cluster_id + '-'
        zk_path_pfx = args.cluster_id + '/'
    else:
        client_pfx = ''
        zk_path_pfx = ''

    # randomize collector list
    args.random_collectors = args.collectors
    if args.collectors:
        args.random_collectors = random.sample(args.collectors,
                                           len(args.collectors))

    km_logger = logger.KubeManagerLogger(args)

    if args.nested_mode == '0':
        # Initialize AMQP handler then close it to be sure remain queue of a
        # precedent run is cleaned
        try:
            vnc_amqp = VncAmqpHandle(km_logger, DBBaseKM,
                                     REACTION_MAP, 'kube_manager', args=args)
            vnc_amqp.establish()
            vnc_amqp.close()
        except Exception:
            pass
        finally:
            km_logger.debug("Removed remained AMQP queue")

        # Ensure zookeeper is up and running before starting kube-manager
        _zookeeper_client = ZookeeperClient(client_pfx+"kube-manager",
                                            args.zk_server_ip)

        km_logger.notice("Waiting to be elected as master...")
        _zookeeper_client.master_election(zk_path_pfx+"/kube-manager",
                                          os.getpid(), run_kube_manager,
                                          km_logger, args, kube_api_skip,
                                          event_queue)
        km_logger.notice("Elected master kube-manager node. Initializing...")

    else: #nested mode, skip zookeeper mastership check
        run_kube_manager(km_logger, args, kube_api_skip, event_queue)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>
import ast
import errno
import os
import platform
import shutil
import subprocess
import sys

from distutils import log
from distutils.command.build_ext import build_ext
from distutils.command.sdist import sdist
from distutils.errors import DistutilsError

def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno==errno.EEXIST and os.path.isdir(path) and os.access(path, os.R_OK | os.W_OK | os.X_OK):
            return
        raise

def rmtree(path):
    try:
        shutil.rmtree(path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

def exec_process(cmdline, silent=True, input=None, **kwargs):
    """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer.
    Optionally prints stdout and stderr while running."""
    try:
        sub = subprocess.Popen(args=cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
        stdout, stderr = sub.communicate(input=input)
        returncode = sub.returncode
        if not silent:
            sys.stdout.write(stdout)
            sys.stderr.write(stderr)
    except OSError as e:
        if e.errno == errno.ENOENT:
            raise DistutilsError('"%s" is not present on this system' % cmdline[0])
        else:
            raise
    if returncode != 0:
        raise DistutilsError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.decode("utf-8").rstrip("\n")))
    return stdout

def prepare_windows_env(env):
    env.pop('VS110COMNTOOLS', None)
    if sys.version_info < (3, 3):
        env.pop('VS100COMNTOOLS', None)

    if env.get('PYTHON'):
        return  # Already manually set by user.

    if (2, 6) <= sys.version_info[:2] <= (2, 7):
        env['PYTHON'] = sys.executable
        return  # The current executable is fine.

    # Try if `python` on PATH is the right one. If we would execute
    # `python` directly the current executable might be used so we
    # delegate this to cmd.
    cmd = ['cmd.exe', '/C',  'python', '-c', 'import sys; '
           'v = str(sys.version_info[:2]); sys.stdout.write(v); '
           'sys.stdout.flush()']
    try:
        sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        stdout, _ = sub.communicate()
        version = ast.literal_eval(stdout.decode('utf-8').strip())
        if (2, 6) <= version <= (2, 7):
            return  # Python on PATH is fine
    except OSError:
        pass

    # Check default install locations
    for v in ('26', '27'):
        path = os.path.join('%SYSTEMDRIVE%', 'Python%s' % v, 'python.exe')
        path = os.path.expandvars(path)
        if os.path.isfile(path):
            log.info('Using "%s" to build libuv...' % path)
            env['PYTHON'] = path
            return

    raise DistutilsError('No appropriate Python version found. An '
                         'installation of 2.6 or 2.7 is required to '
                         'build libuv. You can set the environment '
                         'variable "PYTHON" to point to a custom '
                         'installation location.')

class libuv_build_ext(build_ext):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = 'https://github.com/joyent/libuv.git'
    libuv_branch   = 'v0.10'
    libuv_revision = '2400716'
    libuv_patches  = []

    user_options = build_ext.user_options
    user_options.extend([
        ("libuv-clean-compile", None, "Clean libuv tree before compilation"),
        ("libuv-force-fetch", None, "Remove libuv (if present) and fetch it again")
    ])
    boolean_options = build_ext.boolean_options
    boolean_options.extend(["libuv-clean-compile", "libuv-force-fetch"])

    def initialize_options(self):
        build_ext.initialize_options(self)
        self.libuv_clean_compile = 0
        self.libuv_force_fetch = 0

    def build_extensions(self):
        if self.compiler.compiler_type == 'mingw32':
            # Dirty hack to avoid linking with more than one C runtime when using MinGW
            # Note that this hack forces the compilation to use the old MSVCT
            self.compiler.dll_libraries = [lib for lib in self.compiler.dll_libraries if not lib.startswith('msvcr')]
        self.force = self.libuv_force_fetch or self.libuv_clean_compile
        if self.compiler.compiler_type == 'msvc':
            self.libuv_lib = os.path.join(self.libuv_dir, 'Release', 'lib', 'libuv.lib')
        else:
            self.libuv_lib = os.path.join(self.libuv_dir, 'libuv.a')
        self.get_libuv()
        # Set compiler options
        if self.compiler.compiler_type == 'mingw32':
            self.compiler.add_library_dir(self.libuv_dir)
            self.compiler.add_library('uv')
        else:
            self.extensions[0].extra_objects.extend([self.libuv_lib])
        self.compiler.add_include_dir(os.path.join(self.libuv_dir, 'include'))
        if sys.platform.startswith('linux'):
            self.compiler.add_library('rt')
        elif sys.platform == 'darwin':
            self.extensions[0].extra_link_args.extend(['-framework', 'CoreServices'])
        elif sys.platform == 'win32':
            self.extensions[0].define_macros.append(('WIN32', 1))
            if self.compiler.compiler_type == 'msvc':
                self.extensions[0].extra_link_args.extend(['/NODEFAULTLIB:libcmt', '/LTCG'])
                self.compiler.add_library('advapi32')
            self.compiler.add_library('iphlpapi')
            self.compiler.add_library('psapi')
            self.compiler.add_library('ws2_32')
        build_ext.build_extensions(self)

    def get_libuv(self):
        #self.debug_mode =  bool(self.debug) or hasattr(sys, 'gettotalrefcount')
        win32_msvc = self.compiler.compiler_type=='msvc'
        def download_libuv():
            log.info('Downloading libuv...')
            makedirs(self.libuv_dir)
            exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
            exec_process(['git', 'reset', '--hard', self.libuv_revision], cwd=self.libuv_dir)
        def patch_libuv():
            if self.libuv_patches:
                log.info('Patching libuv...')
                for patch_file in self.libuv_patches:
                    exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        def build_libuv():
            cflags = '-fPIC'
            env = os.environ.copy()
            env['CFLAGS'] = ' '.join(x for x in (cflags, env.get('CFLAGS', None)) if x)
            log.info('Building libuv...')
            if win32_msvc:
                prepare_windows_env(env)
                libuv_arch = {'32bit': 'x86', '64bit': 'x64'}[platform.architecture()[0]]
                exec_process(['cmd.exe', '/C', 'vcbuild.bat', libuv_arch, 'release'], cwd=self.libuv_dir, env=env, shell=True)
            else:
                exec_process(['make', 'libuv.a'], cwd=self.libuv_dir, env=env)
        if self.libuv_force_fetch:
            rmtree('deps')
        if not os.path.exists(self.libuv_dir):
            try:
                download_libuv()
            except BaseException:
                rmtree('deps')
                raise
            patch_libuv()
            build_libuv()
        else:
            if self.libuv_clean_compile:
                if win32_msvc:
                    env = os.environ.copy()
                    prepare_windows_env(env)
                    exec_process(['cmd.exe', '/C', 'vcbuild.bat', 'clean'], cwd=self.libuv_dir, env=env, shell=True)
                    rmtree(os.path.join(self.libuv_dir, 'Release'))
                else:
                    exec_process(['make', 'clean'], cwd=self.libuv_dir)
            if not os.path.exists(self.libuv_lib):
                log.info('libuv needs to be compiled.')
                build_libuv()
            else:
                log.info('No need to build libuv.')

class libuv_sdist(sdist):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = libuv_build_ext.libuv_repo
    libuv_branch   = libuv_build_ext.libuv_branch
    libuv_revision = libuv_build_ext.libuv_revision
    libuv_patches  = libuv_build_ext.libuv_patches

    def initialize_options(self):
        sdist.initialize_options(self)
        rmtree('deps')
        makedirs(self.libuv_dir)
        log.info('Downloading libuv...')
        exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
        exec_process(['git', 'checkout', self.libuv_revision], cwd=self.libuv_dir)
        if self.libuv_patches:
            log.info('Patching libuv...')
            for patch_file in self.libuv_patches:
                exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        rmtree(os.path.join(self.libuv_dir, '.git'))

<commit_msg>Raised libuv revision<commit_after>
import ast
import errno
import os
import platform
import shutil
import subprocess
import sys

from distutils import log
from distutils.command.build_ext import build_ext
from distutils.command.sdist import sdist
from distutils.errors import DistutilsError

def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno==errno.EEXIST and os.path.isdir(path) and os.access(path, os.R_OK | os.W_OK | os.X_OK):
            return
        raise

def rmtree(path):
    try:
        shutil.rmtree(path)
    except OSError as e:
        if e.errno != errno.ENOENT:
            raise

def exec_process(cmdline, silent=True, input=None, **kwargs):
    """Execute a subprocess and returns the returncode, stdout buffer and stderr buffer.
    Optionally prints stdout and stderr while running."""
    try:
        sub = subprocess.Popen(args=cmdline, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
        stdout, stderr = sub.communicate(input=input)
        returncode = sub.returncode
        if not silent:
            sys.stdout.write(stdout)
            sys.stderr.write(stderr)
    except OSError as e:
        if e.errno == errno.ENOENT:
            raise DistutilsError('"%s" is not present on this system' % cmdline[0])
        else:
            raise
    if returncode != 0:
        raise DistutilsError('Got return value %d while executing "%s", stderr output was:\n%s' % (returncode, " ".join(cmdline), stderr.decode("utf-8").rstrip("\n")))
    return stdout

def prepare_windows_env(env):
    env.pop('VS110COMNTOOLS', None)
    if sys.version_info < (3, 3):
        env.pop('VS100COMNTOOLS', None)

    if env.get('PYTHON'):
        return  # Already manually set by user.

    if (2, 6) <= sys.version_info[:2] <= (2, 7):
        env['PYTHON'] = sys.executable
        return  # The current executable is fine.

    # Try if `python` on PATH is the right one. If we would execute
    # `python` directly the current executable might be used so we
    # delegate this to cmd.
    cmd = ['cmd.exe', '/C',  'python', '-c', 'import sys; '
           'v = str(sys.version_info[:2]); sys.stdout.write(v); '
           'sys.stdout.flush()']
    try:
        sub = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        stdout, _ = sub.communicate()
        version = ast.literal_eval(stdout.decode('utf-8').strip())
        if (2, 6) <= version <= (2, 7):
            return  # Python on PATH is fine
    except OSError:
        pass

    # Check default install locations
    for v in ('26', '27'):
        path = os.path.join('%SYSTEMDRIVE%', 'Python%s' % v, 'python.exe')
        path = os.path.expandvars(path)
        if os.path.isfile(path):
            log.info('Using "%s" to build libuv...' % path)
            env['PYTHON'] = path
            return

    raise DistutilsError('No appropriate Python version found. An '
                         'installation of 2.6 or 2.7 is required to '
                         'build libuv. You can set the environment '
                         'variable "PYTHON" to point to a custom '
                         'installation location.')

class libuv_build_ext(build_ext):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = 'https://github.com/joyent/libuv.git'
    libuv_branch   = 'v0.10'
    libuv_revision = '9b801d5'
    libuv_patches  = []

    user_options = build_ext.user_options
    user_options.extend([
        ("libuv-clean-compile", None, "Clean libuv tree before compilation"),
        ("libuv-force-fetch", None, "Remove libuv (if present) and fetch it again")
    ])
    boolean_options = build_ext.boolean_options
    boolean_options.extend(["libuv-clean-compile", "libuv-force-fetch"])

    def initialize_options(self):
        build_ext.initialize_options(self)
        self.libuv_clean_compile = 0
        self.libuv_force_fetch = 0

    def build_extensions(self):
        if self.compiler.compiler_type == 'mingw32':
            # Dirty hack to avoid linking with more than one C runtime when using MinGW
            # Note that this hack forces the compilation to use the old MSVCT
            self.compiler.dll_libraries = [lib for lib in self.compiler.dll_libraries if not lib.startswith('msvcr')]
        self.force = self.libuv_force_fetch or self.libuv_clean_compile
        if self.compiler.compiler_type == 'msvc':
            self.libuv_lib = os.path.join(self.libuv_dir, 'Release', 'lib', 'libuv.lib')
        else:
            self.libuv_lib = os.path.join(self.libuv_dir, 'libuv.a')
        self.get_libuv()
        # Set compiler options
        if self.compiler.compiler_type == 'mingw32':
            self.compiler.add_library_dir(self.libuv_dir)
            self.compiler.add_library('uv')
        else:
            self.extensions[0].extra_objects.extend([self.libuv_lib])
        self.compiler.add_include_dir(os.path.join(self.libuv_dir, 'include'))
        if sys.platform.startswith('linux'):
            self.compiler.add_library('rt')
        elif sys.platform == 'darwin':
            self.extensions[0].extra_link_args.extend(['-framework', 'CoreServices'])
        elif sys.platform == 'win32':
            self.extensions[0].define_macros.append(('WIN32', 1))
            if self.compiler.compiler_type == 'msvc':
                self.extensions[0].extra_link_args.extend(['/NODEFAULTLIB:libcmt', '/LTCG'])
                self.compiler.add_library('advapi32')
            self.compiler.add_library('iphlpapi')
            self.compiler.add_library('psapi')
            self.compiler.add_library('ws2_32')
        build_ext.build_extensions(self)

    def get_libuv(self):
        #self.debug_mode =  bool(self.debug) or hasattr(sys, 'gettotalrefcount')
        win32_msvc = self.compiler.compiler_type=='msvc'
        def download_libuv():
            log.info('Downloading libuv...')
            makedirs(self.libuv_dir)
            exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
            exec_process(['git', 'reset', '--hard', self.libuv_revision], cwd=self.libuv_dir)
        def patch_libuv():
            if self.libuv_patches:
                log.info('Patching libuv...')
                for patch_file in self.libuv_patches:
                    exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        def build_libuv():
            cflags = '-fPIC'
            env = os.environ.copy()
            env['CFLAGS'] = ' '.join(x for x in (cflags, env.get('CFLAGS', None)) if x)
            log.info('Building libuv...')
            if win32_msvc:
                prepare_windows_env(env)
                libuv_arch = {'32bit': 'x86', '64bit': 'x64'}[platform.architecture()[0]]
                exec_process(['cmd.exe', '/C', 'vcbuild.bat', libuv_arch, 'release'], cwd=self.libuv_dir, env=env, shell=True)
            else:
                exec_process(['make', 'libuv.a'], cwd=self.libuv_dir, env=env)
        if self.libuv_force_fetch:
            rmtree('deps')
        if not os.path.exists(self.libuv_dir):
            try:
                download_libuv()
            except BaseException:
                rmtree('deps')
                raise
            patch_libuv()
            build_libuv()
        else:
            if self.libuv_clean_compile:
                if win32_msvc:
                    env = os.environ.copy()
                    prepare_windows_env(env)
                    exec_process(['cmd.exe', '/C', 'vcbuild.bat', 'clean'], cwd=self.libuv_dir, env=env, shell=True)
                    rmtree(os.path.join(self.libuv_dir, 'Release'))
                else:
                    exec_process(['make', 'clean'], cwd=self.libuv_dir)
            if not os.path.exists(self.libuv_lib):
                log.info('libuv needs to be compiled.')
                build_libuv()
            else:
                log.info('No need to build libuv.')

class libuv_sdist(sdist):
    libuv_dir      = os.path.join('deps', 'libuv')
    libuv_repo     = libuv_build_ext.libuv_repo
    libuv_branch   = libuv_build_ext.libuv_branch
    libuv_revision = libuv_build_ext.libuv_revision
    libuv_patches  = libuv_build_ext.libuv_patches

    def initialize_options(self):
        sdist.initialize_options(self)
        rmtree('deps')
        makedirs(self.libuv_dir)
        log.info('Downloading libuv...')
        exec_process(['git', 'clone', '-b', self.libuv_branch, self.libuv_repo, self.libuv_dir])
        exec_process(['git', 'checkout', self.libuv_revision], cwd=self.libuv_dir)
        if self.libuv_patches:
            log.info('Patching libuv...')
            for patch_file in self.libuv_patches:
                exec_process(['patch', '--forward', '-d', self.libuv_dir, '-p0', '-i', os.path.abspath(patch_file)])
        rmtree(os.path.join(self.libuv_dir, '.git'))

<|endoftext|>


<commit_before>
INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',

    '{{ project_name }}',

    'axes',
    # 'djangocms_misc',
    'djangocms_misc.basic',
    # 'djangocms_misc.alternate_toolbar',
    'djangocms_misc.admin_style',
    'djangocms_misc.global_untranslated_placeholder',
    'formfieldstash',
    'ckeditor_link',
    'ckeditor_link.link_model',
    'mailprotector',
    'textblocks',
    # 'filer_addons.filer_utils',
    # 'filer_addons.filer_gui',
    # 'filer_addons.filer_signals',

    # 'djangocms_baseplugins.baseplugin',
    # 'my_baseplugin',
    # 'djangocms_baseplugins.section',
    # 'djangocms_baseplugins.slider',
    # 'djangocms_baseplugins.gallery',
    #
    # 'djangocms_baseplugins.text',
    # 'djangocms_baseplugins.textimage',
    # 'djangocms_baseplugins.image',
    #
    # 'djangocms_baseplugins.teaser',
    # 'djangocms_baseplugins.iframe',
    # 'djangocms_baseplugins.video',

    'compressor',
    'modeltranslation',
    'ckeditor',

    'filer',
    'easy_thumbnails',

    'cms',
    'menus',
    'classytags',
    'treebeard',
    'sekizai',
    'djangocms_admin_style',

    'django.contrib.admin',
]
<commit_msg>danger zone0<commit_after>
INSTALLED_APPS = [
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    # Uncomment the next line to enable admin documentation:
    # 'django.contrib.admindocs',

    '{{ project_name }}',

    'axes',
    # 'djangocms_misc',
    'djangocms_misc.basic',
    # 'djangocms_misc.alternate_toolbar',
    'djangocms_misc.admin_style',
    # 'djangocms_misc.global_untranslated_placeholder',
    'formfieldstash',
    'ckeditor_link',
    'ckeditor_link.link_model',
    'mailprotector',
    'textblocks',
    'filer_addons.filer_utils',
    'filer_addons.filer_gui',
    'filer_addons.filer_signals',

    # 'djangocms_baseplugins.baseplugin',
    # 'my_baseplugin',
    # 'djangocms_baseplugins.section',
    # 'djangocms_baseplugins.slider',
    # 'djangocms_baseplugins.gallery',
    #
    # 'djangocms_baseplugins.text',
    # 'djangocms_baseplugins.textimage',
    # 'djangocms_baseplugins.image',
    #
    # 'djangocms_baseplugins.teaser',
    # 'djangocms_baseplugins.iframe',
    # 'djangocms_baseplugins.video',

    'compressor',
    'modeltranslation',
    'ckeditor',

    'filer',
    'easy_thumbnails',

    'cms',
    'menus',
    'classytags',
    'treebeard',
    'sekizai',
    'djangocms_admin_style',

    'django.contrib.admin',
]
<|endoftext|>


<commit_before>"""
Django settings for dailybruin project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: don't run with debug turned on in production!
# DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = ['localhost', '.dailybruin.com']

# Application definition

DJANGO_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
)

THIRD_PARTY_APPS = (
    'south',
)

LOCAL_APPS = (
    'main',
    'prime',
    'music'
)

INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'project.urls'

WSGI_APPLICATION = 'project.wsgi.application'

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'America/Los_Angeles'

USE_I18N = True

USE_L10N = True

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/static/'
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"

STATIC_ROOT = BASE_DIR + "/../static"
MEDIA_ROOT = BASE_DIR + "/../uploads"

TEMPLATE_DIRS = (
    BASE_DIR+'/templates',
)

<commit_msg>turned off i18n and l10n, changed TEMPLAGE_DIRS to polls tut recommendation<commit_after>"""
Django settings for dailybruin project.

For more information on this file, see
https://docs.djangoproject.com/en/1.6/topics/settings/

For the full list of settings and their values, see
https://docs.djangoproject.com/en/1.6/ref/settings/
"""

# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
BASE_DIR = os.path.dirname(os.path.dirname(__file__))

# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/

# SECURITY WARNING: don't run with debug turned on in production!
# DEBUG = True

TEMPLATE_DEBUG = True

ALLOWED_HOSTS = ['localhost', '.dailybruin.com']

# Application definition

DJANGO_APPS = (
    'django.contrib.admin',
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.messages',
    'django.contrib.staticfiles',
)

THIRD_PARTY_APPS = (
    'south',
)

LOCAL_APPS = (
    'main',
    'prime',
    'music'
)

INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS

MIDDLEWARE_CLASSES = (
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'project.urls'

WSGI_APPLICATION = 'project.wsgi.application'

# Internationalization
# https://docs.djangoproject.com/en/1.6/topics/i18n/

LANGUAGE_CODE = 'en-us'

TIME_ZONE = 'America/Los_Angeles'

USE_I18N = False

USE_TZ = True

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.6/howto/static-files/

STATIC_URL = '/static/'
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"

STATIC_ROOT = BASE_DIR + "/../static"
MEDIA_ROOT = BASE_DIR + "/../uploads"

TEMPLATE_DIRS = [os.path.join(BASE_DIR, 'templates')]
<|endoftext|>