<commit_before>#! /usr/bin/env python
# --*-- coding:utf-8 --*--

import os
import sys
sys.path.append(os.path.split(os.path.abspath(sys.path[0]))[0])

from datetime import datetime
import time
import json
import traceback
from bs4 import BeautifulSoup
import requests
import logging
import logging.config
from decimal import Decimal
from config.setting import *

get_datatime = lambda x : time.strftime(TIME_FORMAT, time.localtime(x))
get_now = lambda : time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime(time.time()))
datetime_timestamp = lambda x : int(time.mktime(time.strptime(x, '%Y-%m-%d %H:%M:%S')))
getTaskFlag = lambda : 'tf{0}'.format(int(time.time()))

def get_unixtime_range():
     start = int(time.mktime(time.strptime(get_now(), '%Y-%m-%dT%H:%M:%S'))) - 120
     end = start + unix_end_offset_seconds
     return start, end

def get_clickid(content):
    soup = BeautifulSoup(content)
    for link in soup.find_all('a'):
        return link.get('href').split('=')[1]

def getLogger():
    logging.config.fileConfig(os.path.join(os.path.split(sys.path[0])[0],'config/logging.conf'))
    logger = logging.getLogger("eredar")
    return logger

def getDruidDetailResult(start_time, end_time, transaction_id_list, realDataCount):
    logger = getLogger()
    param = param_template % (start_time, end_time, transaction_id_list)
    if "'" in param:
        param = param.replace("'",'"')
    geturl = query_url + param
    isEnvOK()
    logger.info('get detail query param: {0}'.format(param))
    logger.info('wait {0} seconds that flow data to druid'.format(timewaitquerydetail))
    time.sleep(timewaitquerydetail)
    try:
        r = requests.get(geturl, timeout=60)
        data = json.loads(r.text).get('data').get('data')
        if realDataCount != len(data[1:]):
            logger.error('imitate data count: {0} != query detail data count: {1}'.format(realDataCount, len(data[1:])))
            sys.exit()
        else:
            logger.info('imitate data count: {0} equal to query detail data count: {1}'.format(realDataCount, len(data[1:])))
        return data[0], data[1:]
    except Exception as ex:
        logger.error('get druid detail result failed: {0}'.format(ex))
        sys.exit()
    else:
        return None

def getResponse(param):
    logger = getLogger()
    url = query_url + param
    try:
        r = requests.get(url)
    except Exception as ex:
        logger.error('send query url get a exception: {0}'.format(ex))
    else:
        return r.text

def unicode2str(unicodeList):
    strList = list()
    if unicodeList:
        for u in unicodeList:
            if not isinstance(u, basestring):
                strList.append(u)
            else:
                strList.append(u.encode())
    return tuple(strList)

def getUtcHour():
    currentHour = datetime.now().hour
    if currentHour >= 8:
        return currentHour - 8
    else:
        return 16 + currentHour

def getStrList(valueList):
    strValueList = list()
    for v in valueList:
        if v == datetime.now().year or v == getUtcHour():
            strValueList.append(str(v))
        elif isinstance(v, Decimal):
            if str(v).find('.') != -1:
                strValueList.append(float(v))
            else:
                strValueList.append(v)
        else:
            strValueList.append(v)
    return strValueList

def getVaildColumn(column):
    if isinstance(column, list):
        columnString = ','.join(column)
    return '({0})'.format(columnString)

def setCache():
    logger = getLogger()
    result = os.popen(cachecmd).read()
    if result.startswith("success"):
        logger.info('reset l1 cache and l2 cache false success')
    result = os.popen(routercmd).read()
    if result.startswith("successfully"):
        logger.info('reset router to druid success')

def portScanner(machine):
    import socket
    sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sk.settimeout(10)
    try:
        sk.connect(machine)
    except Exception as ex:
        return ex
    sk.close()
    return "OK"

def isEnvOK():
    logger = getLogger()
    for machine in machineList:
        if portScanner(machine) != "OK":
            logger.error('telnet {0} on port {1} failed'.format(machine[0], machine[1]))
            sys.exit()
    logger.info('check envirment success')
<commit_msg>change run query rule to query data count == set data count == len(offer_id_aff_id_list) * cycletime * 2<commit_after>#! /usr/bin/env python
# --*-- coding:utf-8 --*--

import os
import sys
sys.path.append(os.path.split(os.path.abspath(sys.path[0]))[0])

from datetime import datetime
import time
import json
import traceback
from bs4 import BeautifulSoup
import requests
import logging
import logging.config
from decimal import Decimal
from config.setting import *

get_datatime = lambda x : time.strftime(TIME_FORMAT, time.localtime(x))
get_now = lambda : time.strftime('%Y-%m-%dT%H:%M:%S',time.localtime(time.time()))
datetime_timestamp = lambda x : int(time.mktime(time.strptime(x, '%Y-%m-%d %H:%M:%S')))
getTaskFlag = lambda : 'tf{0}'.format(int(time.time()))

def get_unixtime_range():
     start = int(time.mktime(time.strptime(get_now(), '%Y-%m-%dT%H:%M:%S'))) - 120
     end = start + unix_end_offset_seconds
     return start, end

def get_clickid(content):
    soup = BeautifulSoup(content)
    for link in soup.find_all('a'):
        return link.get('href').split('=')[1]

def getLogger():
    logging.config.fileConfig(os.path.join(os.path.split(sys.path[0])[0],'config/logging.conf'))
    logger = logging.getLogger("eredar")
    return logger

def getDruidDetailResult(start_time, end_time, transaction_id_list, realDataCount):
    logger = getLogger()
    param = param_template % (start_time, end_time, transaction_id_list)
    if "'" in param:
        param = param.replace("'",'"')
    geturl = query_url + param
    isEnvOK()
    logger.info('get detail query param: {0}'.format(param))
    logger.info('wait {0} seconds that flow data to druid'.format(timewaitquerydetail))
    time.sleep(timewaitquerydetail)
    try:
        r = requests.get(geturl, timeout=60)
        data = json.loads(r.text).get('data').get('data')
        if realDataCount == len(data[1:]) == len(offer_aff_combination) * cycletimes * 2:
            logger.info('imitate data count: {0} equal to query detail data count: {1}'.format(realDataCount, len(data[1:])))
            sys.exit()
        else:
            logger.error('imitate data count: {0} != query detail data count: {1}'.format(realDataCount, len(data[1:])))
        return data[0], data[1:]
    except Exception as ex:
        logger.error('get druid detail result failed: {0}'.format(ex))
        sys.exit()
    else:
        return None

def getResponse(param):
    logger = getLogger()
    url = query_url + param
    try:
        r = requests.get(url)
    except Exception as ex:
        logger.error('send query url get a exception: {0}'.format(ex))
    else:
        return r.text

def unicode2str(unicodeList):
    strList = list()
    if unicodeList:
        for u in unicodeList:
            if not isinstance(u, basestring):
                strList.append(u)
            else:
                strList.append(u.encode())
    return tuple(strList)

def getUtcHour():
    currentHour = datetime.now().hour
    if currentHour >= 8:
        return currentHour - 8
    else:
        return 16 + currentHour

def getStrList(valueList):
    strValueList = list()
    for v in valueList:
        if v == datetime.now().year or v == getUtcHour():
            strValueList.append(str(v))
        elif isinstance(v, Decimal):
            if str(v).find('.') != -1:
                strValueList.append(float(v))
            else:
                strValueList.append(v)
        else:
            strValueList.append(v)
    return strValueList

def getVaildColumn(column):
    if isinstance(column, list):
        columnString = ','.join(column)
    return '({0})'.format(columnString)

def setCache():
    logger = getLogger()
    result = os.popen(cachecmd).read()
    if result.startswith("success"):
        logger.info('reset l1 cache and l2 cache false success')
    result = os.popen(routercmd).read()
    if result.startswith("successfully"):
        logger.info('reset router to druid success')

def portScanner(machine):
    import socket
    sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sk.settimeout(10)
    try:
        sk.connect(machine)
    except Exception as ex:
        return ex
    sk.close()
    return "OK"

def isEnvOK():
    logger = getLogger()
    for machine in machineList:
        if portScanner(machine) != "OK":
            logger.error('telnet {0} on port {1} failed'.format(machine[0], machine[1]))
            sys.exit()
    logger.info('check envirment success')
<|endoftext|>


<commit_before>"""Copyright 2009 Chris Davis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License."""

import traceback
from django.http import HttpResponse, HttpResponseBadRequest
from django.conf import settings
from graphite.account.models import Profile
from graphite.util import getProfile, getProfileByUsername, json
from graphite.logger import log
from graphite.storage import STORE
from graphite.metrics.search import searcher
from graphite.carbonlink import CarbonLink
import fnmatch, os

try:
  import cPickle as pickle
except ImportError:
  import pickle

def index_json(request):
  jsonp = request.REQUEST.get('jsonp', False)
  matches = []

  for root, dirs, files in os.walk(settings.WHISPER_DIR):
    root = root.replace(settings.WHISPER_DIR, '')
    for basename in files:
      if fnmatch.fnmatch(basename, '*.wsp'):
        matches.append(os.path.join(root, basename))

  for root, dirs, files in os.walk(settings.CERES_DIR):
    root = root.replace(settings.CERES_DIR, '')
    for filename in files:
      if filename == '.ceres-node':
        matches.append(root)

  matches = [
    m
    .replace('.wsp', '')
    .replace('.rrd', '')
    .replace('/', '.')
    .lstrip('.')
    for m in sorted(matches)
  ]
  return json_response_for(request, matches, jsonp=jsonp)

def search_view(request):
  try:
    query = str( request.REQUEST['query'] )
  except:
    return HttpResponseBadRequest(content="Missing required parameter 'query'", content_type="text/plain")
  search_request = {
    'query' : query,
    'max_results' : int( request.REQUEST.get('max_results', 25) ),
    'keep_query_pattern' : int(request.REQUEST.get('keep_query_pattern', 0)),
  }
  #if not search_request['query'].endswith('*'):
  #  search_request['query'] += '*'

  results = sorted(searcher.search(**search_request))
  return json_response_for(request, dict(metrics=results))

def find_view(request):
  "View for finding metrics matching a given pattern"
  profile = getProfile(request)
  format = request.REQUEST.get('format', 'treejson')
  local_only = int( request.REQUEST.get('local', 0) )
  wildcards = int( request.REQUEST.get('wildcards', 0) )
  fromTime = int( request.REQUEST.get('from', -1) )
  untilTime = int( request.REQUEST.get('until', -1) )

  if fromTime == -1:
    fromTime = None
  if untilTime == -1:
    untilTime = None

  automatic_variants = int( request.REQUEST.get('automatic_variants', 0) )

  try:
    query = str( request.REQUEST['query'] )
  except:
    return HttpResponseBadRequest(content="Missing required parameter 'query'", content_type="text/plain")

  if '.' in query:
    base_path = query.rsplit('.', 1)[0] + '.'
  else:
    base_path = ''

  if format == 'completer':
    query = query.replace('..', '*.')
    if not query.endswith('*'):
      query += '*'

    if automatic_variants:
      query_parts = query.split('.')
      for i,part in enumerate(query_parts):
        if ',' in part and '{' not in part:
          query_parts[i] = '{%s}' % part
      query = '.'.join(query_parts)

  try:
    matches = list( STORE.find(query, fromTime, untilTime, local=local_only) )
  except:
    log.exception()
    raise

  log.info('find_view query=%s local_only=%s matches=%d' % (query, local_only, len(matches)))
  matches.sort(key=lambda node: node.name)
  log.info("received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d" % (query, fromTime, untilTime, local_only, format, len(matches)))

  if format == 'treejson':
    content = tree_json(matches, base_path, wildcards=profile.advancedUI or wildcards)
    response = json_response_for(request, content)

  elif format == 'pickle':
    content = pickle_nodes(matches)
    response = HttpResponse(content, content_type='application/pickle')

  elif format == 'completer':
    results = []
    for node in matches:
      node_info = dict(path=node.path, name=node.name, is_leaf=str(int(node.is_leaf)))
      if not node.is_leaf:
        node_info['path'] += '.'
      results.append(node_info)

    if len(results) > 1 and wildcards:
      wildcardNode = {'name' : '*'}
      results.append(wildcardNode)

    response = json_response_for(request, { 'metrics' : results})

  else:
    return HttpResponseBadRequest(content="Invalid value for 'format' parameter", content_type="text/plain")

  response['Pragma'] = 'no-cache'
  response['Cache-Control'] = 'no-cache'
  return response

def expand_view(request):
  "View for expanding a pattern into matching metric paths"
  local_only    = int( request.REQUEST.get('local', 0) )
  group_by_expr = int( request.REQUEST.get('groupByExpr', 0) )
  leaves_only   = int( request.REQUEST.get('leavesOnly', 0) )

  results = {}
  for query in request.REQUEST.getlist('query'):
    results[query] = set()
    for node in STORE.find(query, local=local_only):
      if node.is_leaf or not leaves_only:
        results[query].add( node.path )

  # Convert our results to sorted lists because sets aren't json-friendly
  if group_by_expr:
    for query, matches in results.items():
      results[query] = sorted(matches)
  else:
    results = sorted( reduce(set.union, results.values(), set()) )

  result = {
    'results' : results
  }

  response = json_response_for(request, result)
  response['Pragma'] = 'no-cache'
  response['Cache-Control'] = 'no-cache'
  jsonp = request.REQUEST.get('jsonp', False)
  return json_response_for(request, matches, jsonp=jsonp)

def get_metadata_view(request):
  key = request.REQUEST['key']
  metrics = request.REQUEST.getlist('metric')
  results = {}
  for metric in metrics:
    try:
      results[metric] = CarbonLink.get_metadata(metric, key)
    except:
      log.exception()
      results[metric] = dict(error="Unexpected error occurred in CarbonLink.get_metadata(%s, %s)" % (metric, key))

  return json_response_for(request, results)

def set_metadata_view(request):
  results = {}

  if request.method == 'GET':
    metric = request.GET['metric']
    key = request.GET['key']
    value = request.GET['value']
    try:
      results[metric] = CarbonLink.set_metadata(metric, key, value)
    except:
      log.exception()
      results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key))

  elif request.method == 'POST':
    if request.META.get('CONTENT_TYPE') == 'application/json':
      operations = json.loads( request.raw_post_data )
    else:
      operations = json.loads( request.POST['operations'] )

    for op in operations:
      metric = None
      try:
        metric, key, value = op['metric'], op['key'], op['value']
        results[metric] = CarbonLink.set_metadata(metric, key, value)
      except:
        log.exception()
        if metric:
          results[metric] = dict(error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric)

  else:
    results = dict(error="Invalid request method")

  return json_response_for(request, results)

def tree_json(nodes, base_path, wildcards=False):
  results = []

  branchNode = {
    'allowChildren': 1,
    'expandable': 1,
    'leaf': 0,
  }
  leafNode = {
    'allowChildren': 0,
    'expandable': 0,
    'leaf': 1,
  }

  #Add a wildcard node if appropriate
  if len(nodes) > 1 and wildcards:
    wildcardNode = {'text' : '*', 'id' : base_path + '*'}

    if any(not n.is_leaf for n in nodes):
      wildcardNode.update(branchNode)

    else:
      wildcardNode.update(leafNode)

    results.append(wildcardNode)

  found = set()
  results_leaf = []
  results_branch = []
  for node in nodes: #Now let's add the matching children
    if node.name in found:
      continue

    found.add(node.name)
    resultNode = {
      'text' : str(node.name),
      'id' : base_path + str(node.name),
    }

    if node.is_leaf:
      resultNode.update(leafNode)
      results_leaf.append(resultNode)
    else:
      resultNode.update(branchNode)
      results_branch.append(resultNode)

  results.extend(results_branch)
  results.extend(results_leaf)
  return results

def pickle_nodes(nodes):
  nodes_info = []

  for node in nodes:
    info = dict(path=node.path, is_leaf=node.is_leaf)
    if node.is_leaf:
      info['intervals'] = node.intervals

    nodes_info.append(info)

  return pickle.dumps(nodes_info, protocol=-1)

def any(iterable): #python2.4 compatibility
  for i in iterable:
    if i:
      return True
  return False

def json_response_for(request, data, content_type='application/json', jsonp=False, **kwargs):
  accept = request.META.get('HTTP_ACCEPT', 'application/json')
  ensure_ascii = accept == 'application/json'

  content = json.dumps(data, ensure_ascii=ensure_ascii)
  if jsonp:
    content = "%s(%)" % (jsonp, content)
    content_type = 'text/javascript'
  if not ensure_ascii:
    content_type += ';charset=utf-8'

  return HttpResponse(content, content_type=content_type, **kwargs)
<commit_msg>Bug fix for jsonp implementation of metrics/expand.<commit_after>"""Copyright 2009 Chris Davis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License."""

import traceback
from django.http import HttpResponse, HttpResponseBadRequest
from django.conf import settings
from graphite.account.models import Profile
from graphite.util import getProfile, getProfileByUsername, json
from graphite.logger import log
from graphite.storage import STORE
from graphite.metrics.search import searcher
from graphite.carbonlink import CarbonLink
import fnmatch, os

try:
  import cPickle as pickle
except ImportError:
  import pickle

def index_json(request):
  jsonp = request.REQUEST.get('jsonp', False)
  matches = []

  for root, dirs, files in os.walk(settings.WHISPER_DIR):
    root = root.replace(settings.WHISPER_DIR, '')
    for basename in files:
      if fnmatch.fnmatch(basename, '*.wsp'):
        matches.append(os.path.join(root, basename))

  for root, dirs, files in os.walk(settings.CERES_DIR):
    root = root.replace(settings.CERES_DIR, '')
    for filename in files:
      if filename == '.ceres-node':
        matches.append(root)

  matches = [
    m
    .replace('.wsp', '')
    .replace('.rrd', '')
    .replace('/', '.')
    .lstrip('.')
    for m in sorted(matches)
  ]
  return json_response_for(request, matches, jsonp=jsonp)

def search_view(request):
  try:
    query = str( request.REQUEST['query'] )
  except:
    return HttpResponseBadRequest(content="Missing required parameter 'query'", content_type="text/plain")
  search_request = {
    'query' : query,
    'max_results' : int( request.REQUEST.get('max_results', 25) ),
    'keep_query_pattern' : int(request.REQUEST.get('keep_query_pattern', 0)),
  }
  #if not search_request['query'].endswith('*'):
  #  search_request['query'] += '*'

  results = sorted(searcher.search(**search_request))
  return json_response_for(request, dict(metrics=results))

def find_view(request):
  "View for finding metrics matching a given pattern"
  profile = getProfile(request)
  format = request.REQUEST.get('format', 'treejson')
  local_only = int( request.REQUEST.get('local', 0) )
  wildcards = int( request.REQUEST.get('wildcards', 0) )
  fromTime = int( request.REQUEST.get('from', -1) )
  untilTime = int( request.REQUEST.get('until', -1) )

  if fromTime == -1:
    fromTime = None
  if untilTime == -1:
    untilTime = None

  automatic_variants = int( request.REQUEST.get('automatic_variants', 0) )

  try:
    query = str( request.REQUEST['query'] )
  except:
    return HttpResponseBadRequest(content="Missing required parameter 'query'", content_type="text/plain")

  if '.' in query:
    base_path = query.rsplit('.', 1)[0] + '.'
  else:
    base_path = ''

  if format == 'completer':
    query = query.replace('..', '*.')
    if not query.endswith('*'):
      query += '*'

    if automatic_variants:
      query_parts = query.split('.')
      for i,part in enumerate(query_parts):
        if ',' in part and '{' not in part:
          query_parts[i] = '{%s}' % part
      query = '.'.join(query_parts)

  try:
    matches = list( STORE.find(query, fromTime, untilTime, local=local_only) )
  except:
    log.exception()
    raise

  log.info('find_view query=%s local_only=%s matches=%d' % (query, local_only, len(matches)))
  matches.sort(key=lambda node: node.name)
  log.info("received remote find request: pattern=%s from=%s until=%s local_only=%s format=%s matches=%d" % (query, fromTime, untilTime, local_only, format, len(matches)))

  if format == 'treejson':
    content = tree_json(matches, base_path, wildcards=profile.advancedUI or wildcards)
    response = json_response_for(request, content)

  elif format == 'pickle':
    content = pickle_nodes(matches)
    response = HttpResponse(content, content_type='application/pickle')

  elif format == 'completer':
    results = []
    for node in matches:
      node_info = dict(path=node.path, name=node.name, is_leaf=str(int(node.is_leaf)))
      if not node.is_leaf:
        node_info['path'] += '.'
      results.append(node_info)

    if len(results) > 1 and wildcards:
      wildcardNode = {'name' : '*'}
      results.append(wildcardNode)

    response = json_response_for(request, { 'metrics' : results})

  else:
    return HttpResponseBadRequest(content="Invalid value for 'format' parameter", content_type="text/plain")

  response['Pragma'] = 'no-cache'
  response['Cache-Control'] = 'no-cache'
  return response

def expand_view(request):
  "View for expanding a pattern into matching metric paths"
  local_only    = int( request.REQUEST.get('local', 0) )
  group_by_expr = int( request.REQUEST.get('groupByExpr', 0) )
  leaves_only   = int( request.REQUEST.get('leavesOnly', 0) )

  results = {}
  for query in request.REQUEST.getlist('query'):
    results[query] = set()
    for node in STORE.find(query, local=local_only):
      if node.is_leaf or not leaves_only:
        results[query].add( node.path )

  # Convert our results to sorted lists because sets aren't json-friendly
  if group_by_expr:
    for query, matches in results.items():
      results[query] = sorted(matches)
  else:
    results = sorted( reduce(set.union, results.values(), set()) )

  result = {
    'results' : results
  }

  response = json_response_for(request, result)
  response['Pragma'] = 'no-cache'
  response['Cache-Control'] = 'no-cache'
  jsonp = request.REQUEST.get('jsonp', False)
  return json_response_for(request, response, jsonp=jsonp)

def get_metadata_view(request):
  key = request.REQUEST['key']
  metrics = request.REQUEST.getlist('metric')
  results = {}
  for metric in metrics:
    try:
      results[metric] = CarbonLink.get_metadata(metric, key)
    except:
      log.exception()
      results[metric] = dict(error="Unexpected error occurred in CarbonLink.get_metadata(%s, %s)" % (metric, key))

  return json_response_for(request, results)

def set_metadata_view(request):
  results = {}

  if request.method == 'GET':
    metric = request.GET['metric']
    key = request.GET['key']
    value = request.GET['value']
    try:
      results[metric] = CarbonLink.set_metadata(metric, key, value)
    except:
      log.exception()
      results[metric] = dict(error="Unexpected error occurred in CarbonLink.set_metadata(%s, %s)" % (metric, key))

  elif request.method == 'POST':
    if request.META.get('CONTENT_TYPE') == 'application/json':
      operations = json.loads( request.raw_post_data )
    else:
      operations = json.loads( request.POST['operations'] )

    for op in operations:
      metric = None
      try:
        metric, key, value = op['metric'], op['key'], op['value']
        results[metric] = CarbonLink.set_metadata(metric, key, value)
      except:
        log.exception()
        if metric:
          results[metric] = dict(error="Unexpected error occurred in bulk CarbonLink.set_metadata(%s)" % metric)

  else:
    results = dict(error="Invalid request method")

  return json_response_for(request, results)

def tree_json(nodes, base_path, wildcards=False):
  results = []

  branchNode = {
    'allowChildren': 1,
    'expandable': 1,
    'leaf': 0,
  }
  leafNode = {
    'allowChildren': 0,
    'expandable': 0,
    'leaf': 1,
  }

  #Add a wildcard node if appropriate
  if len(nodes) > 1 and wildcards:
    wildcardNode = {'text' : '*', 'id' : base_path + '*'}

    if any(not n.is_leaf for n in nodes):
      wildcardNode.update(branchNode)

    else:
      wildcardNode.update(leafNode)

    results.append(wildcardNode)

  found = set()
  results_leaf = []
  results_branch = []
  for node in nodes: #Now let's add the matching children
    if node.name in found:
      continue

    found.add(node.name)
    resultNode = {
      'text' : str(node.name),
      'id' : base_path + str(node.name),
    }

    if node.is_leaf:
      resultNode.update(leafNode)
      results_leaf.append(resultNode)
    else:
      resultNode.update(branchNode)
      results_branch.append(resultNode)

  results.extend(results_branch)
  results.extend(results_leaf)
  return results

def pickle_nodes(nodes):
  nodes_info = []

  for node in nodes:
    info = dict(path=node.path, is_leaf=node.is_leaf)
    if node.is_leaf:
      info['intervals'] = node.intervals

    nodes_info.append(info)

  return pickle.dumps(nodes_info, protocol=-1)

def any(iterable): #python2.4 compatibility
  for i in iterable:
    if i:
      return True
  return False

def json_response_for(request, data, content_type='application/json', jsonp=False, **kwargs):
  accept = request.META.get('HTTP_ACCEPT', 'application/json')
  ensure_ascii = accept == 'application/json'

  content = json.dumps(data, ensure_ascii=ensure_ascii)
  if jsonp:
    content = "%s(%)" % (jsonp, content)
    content_type = 'text/javascript'
  if not ensure_ascii:
    content_type += ';charset=utf-8'

  return HttpResponse(content, content_type=content_type, **kwargs)
<|endoftext|>


<commit_before>#!/usr/bin/env python
##################################################################
# Copyright (c) 2012, Sergej Srepfler <sergej.srepfler@gmail.com>
# February 2012 - March 2012
# Version 0.2.5, Last change on Mar 16, 2012
# This software is distributed under the terms of BSD license.
##################################################################

# Decode radius packet into individual AVPs

from libRadius import *
import sys

if __name__ == "__main__":
    # level for decoding are: DEBUG, INFO, WARNING, ERROR, CRITICAL
    #logging.basicConfig(level=logging.DEBUG)
    LoadDictionary("dictRadius.xml")
    msg=sys.argv[1]
    print "="*30
    H=HDRItem()
    stripHdr(H,msg)
    avps=splitMsgAVPs(H.msg)
    cmd=dictCOMMANDcode2name(H.flags,H.cmd)
    if cmd==ERROR:
        print 'Unknown command',H.cmd
    else:
        print cmd
    print "Hop-by-Hop=",H.HopByHop,"End-to-End=",H.EndToEnd,"ApplicationId=",H.appId
    for avp in avps:
      print "RAW AVP",avp
      print "Decoded AVP",decodeAVP(avp)
    print "-"*30
<commit_msg>fixed radius decode example<commit_after>#!/usr/bin/env python
##################################################################
# Copyright (c) 2012, Sergej Srepfler <sergej.srepfler@gmail.com>
# February 2012 -
# Version 0.3, Last change on Oct 24, 2012
# This software is distributed under the terms of BSD license.
##################################################################

# Decode radius packet into individual AVPs

from libRadius import *
import sys

if __name__ == "__main__":
    # level for decoding are: DEBUG, INFO, WARNING, ERROR, CRITICAL
    #logging.basicConfig(level=logging.DEBUG)
    LoadDictionary("dictRadius.xml")
    msg=sys.argv[1]
    print "="*30
    H=HDRItem()
    stripHdr(H,msg)
    avps=splitMsgAVPs(H.msg)
    print "Code=",H.Code,"Identifier=",H.Identifier,"Authenticator=",H.Authenticator
    print dictCOMMANDcode2name(H.Code)
    print "-"*30
    for avp in avps:
      print "RAW AVP",avp
      print "Decoded AVP",decodeAVP(avp)
    print "-"*30

######################################################
# History
# 0.3 - Oct 24, 2012 - Radius decode initial version<|endoftext|>


<commit_before>import os
import sys
from SteamworksParser import steamworksparser

g_TypeConversionDict = {
    "uint8": "byte",
    "uint16": "ushort",
    "uint32": "uint",
    "uint64": "ulong",

    "char": "string",
    "int32": "int",
    "int64": "long",

    "uint8 *": "IntPtr",
    "const char *": "string",
    "const char **": "IntPtr",

    # This is for CallbackMsg_t
    "HSteamUser": "int"
}

g_CustomPackSize = {
    # Callbacks
    "AvatarImageLoaded_t": "4",
    "FriendRichPresenceUpdate_t": "4",
    "GameConnectedClanChatMsg_t": "4",
    "GameConnectedChatLeave_t": "1",
    "JoinClanChatRoomCompletionResult_t": "4",
    "GameConnectedFriendChatMsg_t": "4",
    "FriendsGetFollowerCount_t": "4",
    "FriendsIsFollowing_t": "4",
    "FriendsEnumerateFollowingList_t": "4",
    "GSClientDeny_t": "4",
    "GSClientKick_t": "4",
    "GSClientGroupStatus_t": "1",
    "GSStatsReceived_t": "4",
    "GSStatsStored_t": "4",
    "P2PSessionConnectFail_t": "1",
    "SocketStatusCallback_t": "4",
    "ValidateAuthTicketResponse_t": "4",

    # Structs
    "InputAnalogActionData_t": "1",
    "InputDigitalActionData_t": "1",
}

g_SkippedStructs = (
    # Lingering PS3 stuff.
    "PSNGameBootInviteResult_t",
    "PS3TrophiesInstalled_t",

    # We remap these ISteamController structs to ISteamInput
    "ControllerAnalogActionData_t",
    "ControllerDigitalActionData_t",
    "ControllerMotionData_t",

    # String formatting functions. We just use .ToString() instead.
    "SteamNetworkingIdentityRender",
    "SteamNetworkingIPAddrRender",
    "SteamNetworkingPOPIDRender",

    # CustomType
    "SteamIPAddress_t",
)

g_SequentialStructs = (
    "MatchMakingKeyValuePair_t",
)

g_SpecialFieldTypes = {
    "PersonaStateChange_t": {
        "m_nChangeFlags": "EPersonaChange"
    },

    "HTML_NeedsPaint_t": {
        "pBGRA": "IntPtr"
    },

    # These two are returned by a function and the struct needs to be blittable.
    "InputAnalogActionData_t": {
        "bActive": "byte", # Originally bool
    },

    "InputDigitalActionData_t": {
        "bState":  "byte", # Originally bool
        "bActive": "byte", # Originally bool
    },

    "SteamDatagramGameCoordinatorServerLogin": {
        "m_appData": "byte", # Originally char
    },
}

g_ExplicitStructs = {
    "UserStatsReceived_t": {
        "m_nGameID" : "0",
        "m_eResult" : "8",
        "m_steamIDUser" : "12",
    }
}

def main(parser):
    try:
        os.makedirs("autogen/")
    except OSError:
        pass

    lines = []
    callbacklines = []
    for f in parser.files:
        for struct in f.structs:
            lines.extend(parse(struct))
        for callback in f.callbacks:
            callbacklines.extend(parse(callback))

    with open("autogen/SteamStructs.cs", "wb") as out:
        with open("templates/header.txt", "r") as f:
            out.write(bytes(f.read(), "utf-8"))
        out.write(bytes("namespace Steamworks {\n", "utf-8"))
        for line in lines:
            out.write(bytes(line + "\n", "utf-8"))
        out.write(bytes("}\n\n", "utf-8"))
        out.write(bytes("#endif // !DISABLESTEAMWORKS\n", "utf-8"))

    with open("autogen/SteamCallbacks.cs", "wb") as out:
        with open("templates/header.txt", "r") as f:
            out.write(bytes(f.read(), "utf-8"))
        out.write(bytes("namespace Steamworks {\n", "utf-8"))
        for line in callbacklines:
            out.write(bytes(line + "\n", "utf-8"))
        out.write(bytes("}\n\n", "utf-8"))
        out.write(bytes("#endif // !DISABLESTEAMWORKS\n", "utf-8"))

def parse(struct):
    if struct.name in g_SkippedStructs:
        return []

    lines = []
    for comment in struct.c.rawprecomments:
        if type(comment) is steamworksparser.BlankLine:
            continue
        lines.append("\t" + comment)

    structname = struct.name

    packsize = g_CustomPackSize.get(structname, "Packsize.value")
    if g_ExplicitStructs.get(structname, False):
        lines.append("\t[StructLayout(LayoutKind.Explicit, Pack = " + packsize + ")]")
    elif struct.packsize:
        customsize = ""
        if len(struct.fields) == 0:
            customsize = ", Size = 1"
        lines.append("\t[StructLayout(LayoutKind.Sequential, Pack = " + packsize + customsize + ")]")

    if struct.callbackid:
        lines.append("\t[CallbackIdentity(Constants." + struct.callbackid + ")]")

    for name in g_SequentialStructs:
        if name == structname:
            lines.append("\t[StructLayout(LayoutKind.Sequential)]")
            break

    lines.append("\tpublic struct " + structname + " {")

    lines.extend(insert_constructors(structname))

    if struct.callbackid:
        lines.append("\t\tpublic const int k_iCallback = Constants." + struct.callbackid + ";")

    for field in struct.fields:
        lines.extend(parse_field(field, structname))

    if struct.endcomments:
        for comment in struct.endcomments.rawprecomments:
            if type(comment) is steamworksparser.BlankLine:
                lines.append("\t\t")
            else:
                lines.append("\t" + comment)

    lines.append("\t}")
    lines.append("")

    return lines

def parse_field(field, structname):
    lines = []
    for comment in field.c.rawprecomments:
        if type(comment) is steamworksparser.BlankLine:
            lines.append("\t\t")
        else:
            lines.append("\t" + comment)

    fieldtype = g_TypeConversionDict.get(field.type, field.type)
    fieldtype = g_SpecialFieldTypes.get(structname, dict()).get(field.name, fieldtype)

    explicit = g_ExplicitStructs.get(structname, False)
    if explicit:
        lines.append("\t\t[FieldOffset(" + explicit[field.name] + ")]")

    if field.arraysize:
        constantsstr = "Constants."
        if field.arraysize.isdigit():
            constantsstr = ""
        if fieldtype == "string":
            lines.append("\t\t[MarshalAs(UnmanagedType.ByValTStr, SizeConst = " + constantsstr + field.arraysize + ")]")
        else:
            lines.append("\t\t[MarshalAs(UnmanagedType.ByValArray, SizeConst = " + constantsstr + field.arraysize + ")]")
            fieldtype += "[]"

    comment = ""
    if field.c.rawlinecomment:
        comment = field.c.rawlinecomment

    if fieldtype == "bool":
        lines.append("\t\t[MarshalAs(UnmanagedType.I1)]")

    lines.append("\t\tpublic " + fieldtype + " " + field.name + ";" + comment)

    return lines

def insert_constructors(name):
    lines = []
    if name == "MatchMakingKeyValuePair_t":
        lines.append("\t\tMatchMakingKeyValuePair_t(string strKey, string strValue) {")
        lines.append("\t\t\tm_szKey = strKey;")
        lines.append("\t\t\tm_szValue = strValue;")
        lines.append("\t\t}")

    return lines

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("TODO: Usage Instructions")
        exit()

    steamworksparser.Settings.fake_gameserver_interfaces = True
    main(steamworksparser.parse(sys.argv[1]))<commit_msg>Support for utf8 strings inside structs<commit_after>import os
import sys
from SteamworksParser import steamworksparser

g_TypeConversionDict = {
    "uint8": "byte",
    "uint16": "ushort",
    "uint32": "uint",
    "uint64": "ulong",

    "char": "string",
    "int32": "int",
    "int64": "long",

    "uint8 *": "IntPtr",
    "const char *": "string",
    "const char **": "IntPtr",

    # This is for CallbackMsg_t
    "HSteamUser": "int"
}

g_CustomPackSize = {
    # Callbacks
    "AvatarImageLoaded_t": "4",
    "FriendRichPresenceUpdate_t": "4",
    "GameConnectedClanChatMsg_t": "4",
    "GameConnectedChatLeave_t": "1",
    "JoinClanChatRoomCompletionResult_t": "4",
    "GameConnectedFriendChatMsg_t": "4",
    "FriendsGetFollowerCount_t": "4",
    "FriendsIsFollowing_t": "4",
    "FriendsEnumerateFollowingList_t": "4",
    "GSClientDeny_t": "4",
    "GSClientKick_t": "4",
    "GSClientGroupStatus_t": "1",
    "GSStatsReceived_t": "4",
    "GSStatsStored_t": "4",
    "P2PSessionConnectFail_t": "1",
    "SocketStatusCallback_t": "4",
    "ValidateAuthTicketResponse_t": "4",

    # Structs
    "InputAnalogActionData_t": "1",
    "InputDigitalActionData_t": "1",
}

g_SkippedStructs = (
    # Lingering PS3 stuff.
    "PSNGameBootInviteResult_t",
    "PS3TrophiesInstalled_t",

    # We remap these ISteamController structs to ISteamInput
    "ControllerAnalogActionData_t",
    "ControllerDigitalActionData_t",
    "ControllerMotionData_t",

    # String formatting functions. We just use .ToString() instead.
    "SteamNetworkingIdentityRender",
    "SteamNetworkingIPAddrRender",
    "SteamNetworkingPOPIDRender",

    # CustomType
    "SteamIPAddress_t",
)

g_SequentialStructs = (
    "MatchMakingKeyValuePair_t",
)

g_SpecialFieldTypes = {
    "PersonaStateChange_t": {
        "m_nChangeFlags": "EPersonaChange"
    },

    "HTML_NeedsPaint_t": {
        "pBGRA": "IntPtr"
    },

    # These two are returned by a function and the struct needs to be blittable.
    "InputAnalogActionData_t": {
        "bActive": "byte", # Originally bool
    },

    "InputDigitalActionData_t": {
        "bState":  "byte", # Originally bool
        "bActive": "byte", # Originally bool
    },

    "SteamDatagramGameCoordinatorServerLogin": {
        "m_appData": "byte", # Originally char
    },
}

g_ExplicitStructs = {
    "UserStatsReceived_t": {
        "m_nGameID" : "0",
        "m_eResult" : "8",
        "m_steamIDUser" : "12",
    }
}

def main(parser):
    try:
        os.makedirs("autogen/")
    except OSError:
        pass

    lines = []
    callbacklines = []
    for f in parser.files:
        for struct in f.structs:
            lines.extend(parse(struct))
        for callback in f.callbacks:
            callbacklines.extend(parse(callback))

    with open("autogen/SteamStructs.cs", "wb") as out:
        with open("templates/header.txt", "r") as f:
            out.write(bytes(f.read(), "utf-8"))
        out.write(bytes("namespace Steamworks {\n", "utf-8"))
        for line in lines:
            out.write(bytes(line + "\n", "utf-8"))
        out.write(bytes("}\n\n", "utf-8"))
        out.write(bytes("#endif // !DISABLESTEAMWORKS\n", "utf-8"))

    with open("autogen/SteamCallbacks.cs", "wb") as out:
        with open("templates/header.txt", "r") as f:
            out.write(bytes(f.read(), "utf-8"))
        out.write(bytes("namespace Steamworks {\n", "utf-8"))
        for line in callbacklines:
            out.write(bytes(line + "\n", "utf-8"))
        out.write(bytes("}\n\n", "utf-8"))
        out.write(bytes("#endif // !DISABLESTEAMWORKS\n", "utf-8"))

def parse(struct):
    if struct.name in g_SkippedStructs:
        return []

    lines = []
    for comment in struct.c.rawprecomments:
        if type(comment) is steamworksparser.BlankLine:
            continue
        lines.append("\t" + comment)

    structname = struct.name

    packsize = g_CustomPackSize.get(structname, "Packsize.value")
    if g_ExplicitStructs.get(structname, False):
        lines.append("\t[StructLayout(LayoutKind.Explicit, Pack = " + packsize + ")]")
    elif struct.packsize:
        customsize = ""
        if len(struct.fields) == 0:
            customsize = ", Size = 1"
        lines.append("\t[StructLayout(LayoutKind.Sequential, Pack = " + packsize + customsize + ")]")

    if struct.callbackid:
        lines.append("\t[CallbackIdentity(Constants." + struct.callbackid + ")]")

    for name in g_SequentialStructs:
        if name == structname:
            lines.append("\t[StructLayout(LayoutKind.Sequential)]")
            break

    lines.append("\tpublic struct " + structname + " {")

    lines.extend(insert_constructors(structname))

    if struct.callbackid:
        lines.append("\t\tpublic const int k_iCallback = Constants." + struct.callbackid + ";")

    for field in struct.fields:
        lines.extend(parse_field(field, structname))

    if struct.endcomments:
        for comment in struct.endcomments.rawprecomments:
            if type(comment) is steamworksparser.BlankLine:
                lines.append("\t\t")
            else:
                lines.append("\t" + comment)

    lines.append("\t}")
    lines.append("")

    return lines

def parse_field(field, structname):
    lines = []
    for comment in field.c.rawprecomments:
        if type(comment) is steamworksparser.BlankLine:
            lines.append("\t\t")
        else:
            lines.append("\t" + comment)

    fieldtype = g_TypeConversionDict.get(field.type, field.type)
    fieldtype = g_SpecialFieldTypes.get(structname, dict()).get(field.name, fieldtype)

    explicit = g_ExplicitStructs.get(structname, False)
    if explicit:
        lines.append("\t\t[FieldOffset(" + explicit[field.name] + ")]")

    comment = ""
    if field.c.rawlinecomment:
        comment = field.c.rawlinecomment

    if field.arraysize:
        constantsstr = ""
        if not field.arraysize.isdigit():
            constantsstr = "Constants."

        if fieldtype == "byte[]":
            lines.append("\t\t[MarshalAs(UnmanagedType.ByValArray, SizeConst = " + constantsstr + field.arraysize + ")]")
        else:
            lines.append("\t\t[MarshalAs(UnmanagedType.ByValArray, SizeConst = " + constantsstr + field.arraysize + ")]")
            fieldtype += "[]"

    if fieldtype == "bool":
        lines.append("\t\t[MarshalAs(UnmanagedType.I1)]")

    if field.arraysize and fieldtype == "string[]":
        lines.append("\t\tprivate byte[] " + field.name + "_;")
        lines.append("\t\tpublic string " + field.name + comment)
        lines.append("\t\t{")
        lines.append("\t\t\tget { return InteropHelp.ByteArrayToStringUTF8(" + field.name + "_); }")
        lines.append("\t\t\tset { InteropHelp.StringToByteArrayUTF8(value, " + field.name + "_, " + constantsstr + field.arraysize + "); }")
        lines.append("\t\t}")
    else:
        lines.append("\t\tpublic " + fieldtype + " " + field.name + ";" + comment)

    return lines

def insert_constructors(name):
    lines = []
    if name == "MatchMakingKeyValuePair_t":
        lines.append("\t\tMatchMakingKeyValuePair_t(string strKey, string strValue) {")
        lines.append("\t\t\tm_szKey_ = null;")
        lines.append("\t\t\tm_szValue_ = null;")
        lines.append("\t\t\tm_szKey = strKey;")
        lines.append("\t\t\tm_szValue = strValue;")
        lines.append("\t\t}")

    return lines

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("TODO: Usage Instructions")
        exit()

    steamworksparser.Settings.fake_gameserver_interfaces = True
    main(steamworksparser.parse(sys.argv[1]))<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

import os
import sys
import time
import httplib
import unittest
import threading

sys.path.append('.')

import marionette_tg.conf

def execute(cmd):
    os.system(cmd)

def exec_download():
    client_listen_ip = marionette_tg.conf.get("client.client_ip")
    conn = httplib.HTTPConnection(
        client_listen_ip, 18079, False, timeout=30)
    conn.request("GET", "/")
    response = conn.getresponse()
    actual_response = response.read()
    conn.close()

    expected_response = ''
    for x in range(2**18):
        expected_response += '_' + str(x)

    assert actual_response == expected_response

    return actual_response

class ParametrizedTestCase(unittest.TestCase):
    """ TestCase classes that want to be parametrized should
        inherit from this class.
    """
    def __init__(self, methodName='runTest', param=None):
        super(ParametrizedTestCase, self).__init__(methodName)
        self.param = param

    @staticmethod
    def parametrize(testcase_klass, param=None):
        """ Create a suite containing all tests taken from the given
            subclass, passing them the parameter 'param'.
        """
        testloader = unittest.TestLoader()
        testnames = testloader.getTestCaseNames(testcase_klass)
        suite = unittest.TestSuite()
        for name in testnames:
            suite.addTest(testcase_klass(name, param=param))
        return suite

class CliTest(ParametrizedTestCase):

    def startservers(self, format):
        client_listen_ip = marionette_tg.conf.get("client.client_ip")
        server_proxy_ip = marionette_tg.conf.get("server.proxy_ip")

        execute("./bin/httpserver --local_port 18081 &")
        execute("./bin/marionette_server --proxy_ip %s --proxy_port 18081 --format %s &" %
                (server_proxy_ip, format))
        time.sleep(5)
        execute("./bin/marionette_client --client_ip %s --client_port 18079 --format %s &" %
                (client_listen_ip, format))
        time.sleep(5)

    def stopservers(self):
        execute("pkill -9 -f marionette_client")
        execute("pkill -9 -f marionette_server")
        execute("pkill -9 -f httpserver")

    def dodownload_serial(self):
        exec_download()

    def dodownload_parallel(self):
        simultaneous = 2
        threads = []
        for j in range(simultaneous):
            t = threading.Thread(target=exec_download)
            threads.append(t)
        for t in threads:
            t.start()
        for t in threads:
            t.join()

    def test_cli_curl(self):
        if self.param:
            try:
                format = self.param
                self.startservers(format)
                self.dodownload_serial()
                self.dodownload_parallel()
                sys.stdout.write(format+' ')
                sys.stdout.flush()
            except Exception as e:
                self.assertFalse(True, e)
            finally:
                self.stopservers()

suite = unittest.TestSuite()
for param in [
        'dummy',
        'http_timings',
        'ftp_simple_blocking',
        'http_simple_blocking',
        'http_simple_blocking:20150701', # tests in-band nego.
        'http_simple_blocking:20150702', # tests in-band nego.
        'http_squid_blocking',
        'http_simple_nonblocking',
        'http_probabilistic_blocking',
        'http_simple_blocking_with_msg_lens'
        'ssh_simple_nonblocking',
        'smb_simple_nonblocking',
        'http_active_probing',
        'http_active_probing2',
        'active_probing/http_apache_247',
        'active_probing/ssh_openssh_661',
        'active_probing/ftp_pureftpd_10']:
        suite.addTest(ParametrizedTestCase.parametrize(CliTest, param=param))
unittest.TextTestRunner(verbosity=2).run(suite)
<commit_msg>Update cli_tests.py<commit_after>#!/usr/bin/env python
# coding: utf-8

import os
import sys
import time
import httplib
import unittest
import threading

sys.path.append('.')

import marionette_tg.conf

def execute(cmd):
    os.system(cmd)

def exec_download():
    client_listen_ip = marionette_tg.conf.get("client.client_ip")
    conn = httplib.HTTPConnection(
        client_listen_ip, 18079, False, timeout=30)
    conn.request("GET", "/")
    response = conn.getresponse()
    actual_response = response.read()
    conn.close()

    expected_response = ''
    for x in range(2**18):
        expected_response += '_' + str(x)

    assert actual_response == expected_response

    return actual_response

class ParametrizedTestCase(unittest.TestCase):
    """ TestCase classes that want to be parametrized should
        inherit from this class.
    """
    def __init__(self, methodName='runTest', param=None):
        super(ParametrizedTestCase, self).__init__(methodName)
        self.param = param

    @staticmethod
    def parametrize(testcase_klass, param=None):
        """ Create a suite containing all tests taken from the given
            subclass, passing them the parameter 'param'.
        """
        testloader = unittest.TestLoader()
        testnames = testloader.getTestCaseNames(testcase_klass)
        suite = unittest.TestSuite()
        for name in testnames:
            suite.addTest(testcase_klass(name, param=param))
        return suite

class CliTest(ParametrizedTestCase):

    def startservers(self, format):
        client_listen_ip = marionette_tg.conf.get("client.client_ip")
        server_proxy_ip = marionette_tg.conf.get("server.proxy_ip")

        execute("./bin/httpserver --local_port 18081 &")
        execute("./bin/marionette_server --proxy_ip %s --proxy_port 18081 --format %s &" %
                (server_proxy_ip, format))
        time.sleep(5)
        execute("./bin/marionette_client --client_ip %s --client_port 18079 --format %s &" %
                (client_listen_ip, format))
        time.sleep(5)

    def stopservers(self):
        execute("pkill -9 -f marionette_client")
        execute("pkill -9 -f marionette_server")
        execute("pkill -9 -f httpserver")

    def dodownload_serial(self):
        exec_download()

    def dodownload_parallel(self):
        simultaneous = 2
        threads = []
        for j in range(simultaneous):
            t = threading.Thread(target=exec_download)
            threads.append(t)
        for t in threads:
            t.start()
        for t in threads:
            t.join()

    def test_cli_curl(self):
        if self.param:
            try:
                format = self.param
                self.startservers(format)
                self.dodownload_serial()
                self.dodownload_parallel()
                sys.stdout.write(format+' ')
                sys.stdout.flush()
            except Exception as e:
                self.assertFalse(True, e)
            finally:
                self.stopservers()

suite = unittest.TestSuite()
for param in [
        'dummy',
        'http_timings',
        'ftp_simple_blocking',
        'http_simple_blocking',
        'http_simple_blocking:20150701', # tests in-band nego.
        'http_simple_blocking:20150702', # tests in-band nego.
        'http_squid_blocking',
        'http_simple_nonblocking',
        'http_probabilistic_blocking',
        'http_simple_blocking_with_msg_lens',
        'ssh_simple_nonblocking',
        'smb_simple_nonblocking',
        'http_active_probing',
        'http_active_probing2',
        'active_probing/http_apache_247',
        'active_probing/ssh_openssh_661',
        'active_probing/ftp_pureftpd_10']:
        suite.addTest(ParametrizedTestCase.parametrize(CliTest, param=param))
unittest.TextTestRunner(verbosity=2).run(suite)
<|endoftext|>


<commit_before>"""Copyright 2008 Orbitz WorldWide

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License."""

import pytz
from datetime import datetime,timedelta
from time import daylight
from django.utils import timezone # noqa
from django.conf import settings

months = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec']
weekdays = ['sun','mon','tue','wed','thu','fri','sat']

def parseATTime(s, tzinfo=None):
  if tzinfo is None:
    tzinfo = pytz.timezone(settings.TIME_ZONE)
  s = s.strip().lower().replace('_','').replace(',','').replace(' ','')
  if s.isdigit():
    if len(s) == 8 and int(s[:4]) > 1900 and int(s[4:6]) < 13 and int(s[6:]) < 32:
      pass #Fall back because its not a timestamp, its YYYYMMDD form
    else:
      return datetime.fromtimestamp(int(s),tzinfo)
  elif ':' in s and len(s) == 13:
    return tzinfo.localize(datetime.strptime(s,'%H:%M%Y%m%d'), daylight)
  if '+' in s:
    ref,offset = s.split('+',1)
    offset = '+' + offset
  elif '-' in s:
    ref,offset = s.split('-',1)
    offset = '-' + offset
  else:
    ref,offset = s,''
  return tzinfo.localize((parseTimeReference(ref) + parseTimeOffset(offset)), daylight)

def parseTimeReference(ref):
  if not ref or ref == 'now': return datetime.now()

  #Time-of-day reference
  i = ref.find(':')
  hour,min = 0,0
  if i != -1:
    hour = int( ref[:i] )
    min = int( ref[i+1:i+3] )
    ref = ref[i+3:]
    if ref[:2] == 'am': ref = ref[2:]
    elif ref[:2] == 'pm':
      hour = (hour + 12) % 24
      ref = ref[2:]
  if ref.startswith('noon'):
    hour,min = 12,0
    ref = ref[4:]
  elif ref.startswith('midnight'):
    hour,min = 0,0
    ref = ref[8:]
  elif ref.startswith('teatime'):
    hour,min = 16,0
    ref = ref[7:]

  refDate = datetime.now().replace(hour=hour,minute=min,second=0)

  #Day reference
  if ref in ('yesterday','today','tomorrow'): #yesterday, today, tomorrow
    if ref == 'yesterday':
      refDate = refDate - timedelta(days=1)
    if ref == 'tomorrow':
      refDate = refDate + timedelta(days=1)
  elif ref.count('/') == 2: #MM/DD/YY[YY]
    m,d,y = map(int,ref.split('/'))
    if y < 1900: y += 1900
    if y < 1970: y += 100
    refDate = refDate.replace(year=y)

    try: # Fix for Bug #551771
      refDate = refDate.replace(month=m)
      refDate = refDate.replace(day=d)
    except:
      refDate = refDate.replace(day=d)
      refDate = refDate.replace(month=m)

  elif len(ref) == 8 and ref.isdigit(): #YYYYMMDD
    refDate = refDate.replace(year= int(ref[:4]))

    try: # Fix for Bug #551771
      refDate = refDate.replace(month= int(ref[4:6]))
      refDate = refDate.replace(day= int(ref[6:8]))
    except:
      refDate = refDate.replace(day= int(ref[6:8]))
      refDate = refDate.replace(month= int(ref[4:6]))

  elif ref[:3] in months: #MonthName DayOfMonth
    refDate = refDate.replace(month= months.index(ref[:3]) + 1)
    if ref[-2:].isdigit():
      refDate = refDate.replace(day= int(ref[-2:]))
    elif ref[-1:].isdigit():
      refDate = refDate.replace(day= int(ref[-1:]))
    else:
      raise Exception("Day of month required after month name")
  elif ref[:3] in weekdays: #DayOfWeek (Monday, etc)
    todayDayName = refDate.strftime("%a").lower()[:3]
    today = weekdays.index( todayDayName )
    twoWeeks = weekdays * 2
    dayOffset = today - twoWeeks.index(ref[:3])
    if dayOffset < 0: dayOffset += 7
    refDate -= timedelta(days=dayOffset)
  elif ref:
    raise Exception("Unknown day reference")
  return refDate

def parseTimeOffset(offset):
  if not offset:
    return timedelta()

  t = timedelta()

  if offset[0].isdigit():
    sign = 1
  else:
    sign = { '+' : 1, '-' : -1 }[offset[0]]
    offset = offset[1:]

  while offset:
    i = 1
    while offset[:i].isdigit() and i <= len(offset): i += 1
    num = int(offset[:i-1])
    offset = offset[i-1:]
    i = 1
    while offset[:i].isalpha() and i <= len(offset): i += 1
    unit = offset[:i-1]
    offset = offset[i-1:]
    unitString = getUnitString(unit)
    if unitString == 'months':
      unitString = 'days'
      num = num * 30
    if unitString == 'years':
      unitString = 'days'
      num = num * 365
    t += timedelta(**{ unitString : sign * num})

  return t

def getUnitString(s):
  if s.startswith('s'): return 'seconds'
  if s.startswith('min'): return 'minutes'
  if s.startswith('h'): return 'hours'
  if s.startswith('d'): return 'days'
  if s.startswith('w'): return 'weeks'
  if s.startswith('mon'): return 'months'
  if s.startswith('y'): return 'years'
  raise Exception("Invalid offset unit '%s'" % s)
<commit_msg>django.utils timezone no longer used here<commit_after>"""Copyright 2008 Orbitz WorldWide

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License."""

import pytz
from datetime import datetime,timedelta
from time import daylight
from django.conf import settings

months = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec']
weekdays = ['sun','mon','tue','wed','thu','fri','sat']

def parseATTime(s, tzinfo=None):
  if tzinfo is None:
    tzinfo = pytz.timezone(settings.TIME_ZONE)
  s = s.strip().lower().replace('_','').replace(',','').replace(' ','')
  if s.isdigit():
    if len(s) == 8 and int(s[:4]) > 1900 and int(s[4:6]) < 13 and int(s[6:]) < 32:
      pass #Fall back because its not a timestamp, its YYYYMMDD form
    else:
      return datetime.fromtimestamp(int(s),tzinfo)
  elif ':' in s and len(s) == 13:
    return tzinfo.localize(datetime.strptime(s,'%H:%M%Y%m%d'), daylight)
  if '+' in s:
    ref,offset = s.split('+',1)
    offset = '+' + offset
  elif '-' in s:
    ref,offset = s.split('-',1)
    offset = '-' + offset
  else:
    ref,offset = s,''
  return tzinfo.localize((parseTimeReference(ref) + parseTimeOffset(offset)), daylight)

def parseTimeReference(ref):
  if not ref or ref == 'now': return datetime.now()

  #Time-of-day reference
  i = ref.find(':')
  hour,min = 0,0
  if i != -1:
    hour = int( ref[:i] )
    min = int( ref[i+1:i+3] )
    ref = ref[i+3:]
    if ref[:2] == 'am': ref = ref[2:]
    elif ref[:2] == 'pm':
      hour = (hour + 12) % 24
      ref = ref[2:]
  if ref.startswith('noon'):
    hour,min = 12,0
    ref = ref[4:]
  elif ref.startswith('midnight'):
    hour,min = 0,0
    ref = ref[8:]
  elif ref.startswith('teatime'):
    hour,min = 16,0
    ref = ref[7:]

  refDate = datetime.now().replace(hour=hour,minute=min,second=0)

  #Day reference
  if ref in ('yesterday','today','tomorrow'): #yesterday, today, tomorrow
    if ref == 'yesterday':
      refDate = refDate - timedelta(days=1)
    if ref == 'tomorrow':
      refDate = refDate + timedelta(days=1)
  elif ref.count('/') == 2: #MM/DD/YY[YY]
    m,d,y = map(int,ref.split('/'))
    if y < 1900: y += 1900
    if y < 1970: y += 100
    refDate = refDate.replace(year=y)

    try: # Fix for Bug #551771
      refDate = refDate.replace(month=m)
      refDate = refDate.replace(day=d)
    except:
      refDate = refDate.replace(day=d)
      refDate = refDate.replace(month=m)

  elif len(ref) == 8 and ref.isdigit(): #YYYYMMDD
    refDate = refDate.replace(year= int(ref[:4]))

    try: # Fix for Bug #551771
      refDate = refDate.replace(month= int(ref[4:6]))
      refDate = refDate.replace(day= int(ref[6:8]))
    except:
      refDate = refDate.replace(day= int(ref[6:8]))
      refDate = refDate.replace(month= int(ref[4:6]))

  elif ref[:3] in months: #MonthName DayOfMonth
    refDate = refDate.replace(month= months.index(ref[:3]) + 1)
    if ref[-2:].isdigit():
      refDate = refDate.replace(day= int(ref[-2:]))
    elif ref[-1:].isdigit():
      refDate = refDate.replace(day= int(ref[-1:]))
    else:
      raise Exception("Day of month required after month name")
  elif ref[:3] in weekdays: #DayOfWeek (Monday, etc)
    todayDayName = refDate.strftime("%a").lower()[:3]
    today = weekdays.index( todayDayName )
    twoWeeks = weekdays * 2
    dayOffset = today - twoWeeks.index(ref[:3])
    if dayOffset < 0: dayOffset += 7
    refDate -= timedelta(days=dayOffset)
  elif ref:
    raise Exception("Unknown day reference")
  return refDate

def parseTimeOffset(offset):
  if not offset:
    return timedelta()

  t = timedelta()

  if offset[0].isdigit():
    sign = 1
  else:
    sign = { '+' : 1, '-' : -1 }[offset[0]]
    offset = offset[1:]

  while offset:
    i = 1
    while offset[:i].isdigit() and i <= len(offset): i += 1
    num = int(offset[:i-1])
    offset = offset[i-1:]
    i = 1
    while offset[:i].isalpha() and i <= len(offset): i += 1
    unit = offset[:i-1]
    offset = offset[i-1:]
    unitString = getUnitString(unit)
    if unitString == 'months':
      unitString = 'days'
      num = num * 30
    if unitString == 'years':
      unitString = 'days'
      num = num * 365
    t += timedelta(**{ unitString : sign * num})

  return t

def getUnitString(s):
  if s.startswith('s'): return 'seconds'
  if s.startswith('min'): return 'minutes'
  if s.startswith('h'): return 'hours'
  if s.startswith('d'): return 'days'
  if s.startswith('w'): return 'weeks'
  if s.startswith('mon'): return 'months'
  if s.startswith('y'): return 'years'
  raise Exception("Invalid offset unit '%s'" % s)
<|endoftext|>


<commit_before>from __future__ import division

import numpy as np
import scipy.stats as stats
import scipy as sp
import logging

__all__ = ['histpoints', 'calc_nbins']

def calc_nbins(x, maximum=150):
    n =  (max(x) - min(x)) / (2 * len(x)**(-1/3) * (np.percentile(x, 75) - np.percentile(x, 25)))
    return np.floor(min(n, maximum))

def poisson_limits(N, kind, confidence=0.6827):
    alpha = 1 - confidence
    upper = np.zeros(len(N))
    lower = np.zeros(len(N))
    if kind == 'gamma':
        lower = stats.gamma.ppf(alpha / 2, N)
        upper = stats.gamma.ppf(1 - alpha / 2, N + 1)
    elif kind == 'sqrt':
        lower = sqrt(N)
        upper = lower
    else:
        raise ValueError('Unknown errorbar kind: {}'.format(kind))
    # clip lower bars
    lower[N==0] = 0
    return N - lower, upper - N

def histpoints(x, bins=None, xerr=None, yerr='gamma', normed=False, **kwargs):
    """
    Plot a histogram as a series of data points.

    Compute and draw the histogram of *x* using individual (x,y) points
    for the bin contents.

    By default, vertical poisson error bars are calculated using the
    gamma distribution.

    Horizontal error bars are omitted by default.
    These can be enabled using the *xerr* argument.
    Use ``xerr='binwidth'`` to draw horizontal error bars that indicate
    the width of each histogram bin.

    Parameters
    ---------

    x : (n,) array or sequence of (n,) arrays
        Input values. This takes either a single array or a sequence of
        arrays, which are not required to be of the same length.

    """
    import matplotlib.pyplot as plt

    if bins is None:
        bins = calc_nbins(x)

    h, bins = np.histogram(x, bins=bins)
    width = bins[1] - bins[0]
    center = (bins[:-1] + bins[1:]) / 2
    area = sum(h * width)

    if isinstance(yerr, str):
        yerr = poisson_limits(h, yerr)

    if xerr == 'binwidth':
        xerr = width / 2

    if normed:
        h = h / area
        yerr = yerr / area
        area = 1.

    if not 'color' in kwargs:
        kwargs['color'] = 'black'

    if not 'fmt' in kwargs:
        kwargs['fmt'] = 'o'

    plt.errorbar(center, h, xerr=xerr, yerr=yerr, **kwargs)

    return center, h, area

<commit_msg>Add plot_pull function<commit_after>from __future__ import division

import numpy as np
import scipy.stats as stats
import scipy as sp
import logging

__all__ = ['histpoints', 'make_split', 'calc_nbins', 'plot_pull']

def calc_nbins(x, maximum=150):
    n =  (max(x) - min(x)) / (2 * len(x)**(-1/3) * (np.percentile(x, 75) - np.percentile(x, 25)))
    return np.floor(min(n, maximum))

def poisson_limits(N, kind, confidence=0.6827):
    alpha = 1 - confidence
    upper = np.zeros(len(N))
    lower = np.zeros(len(N))
    if kind == 'gamma':
        lower = stats.gamma.ppf(alpha / 2, N)
        upper = stats.gamma.ppf(1 - alpha / 2, N + 1)
    elif kind == 'sqrt':
        lower = sqrt(N)
        upper = lower
    else:
        raise ValueError('Unknown errorbar kind: {}'.format(kind))
    # clip lower bars
    lower[N==0] = 0
    return N - lower, upper - N

def histpoints(x, bins=None, xerr=None, yerr='gamma', normed=False, **kwargs):
    """
    Plot a histogram as a series of data points.

    Compute and draw the histogram of *x* using individual (x,y) points
    for the bin contents.

    By default, vertical poisson error bars are calculated using the
    gamma distribution.

    Horizontal error bars are omitted by default.
    These can be enabled using the *xerr* argument.
    Use ``xerr='binwidth'`` to draw horizontal error bars that indicate
    the width of each histogram bin.

    Parameters
    ---------

    x : (n,) array or sequence of (n,) arrays
        Input values. This takes either a single array or a sequence of
        arrays, which are not required to be of the same length.

    """
    import matplotlib.pyplot as plt

    if bins is None:
        bins = calc_nbins(x)

    h, bins = np.histogram(x, bins=bins)
    width = bins[1] - bins[0]
    center = (bins[:-1] + bins[1:]) / 2
    area = sum(h * width)

    if isinstance(yerr, str):
        yerr = poisson_limits(h, yerr)

    if xerr == 'binwidth':
        xerr = width / 2

    if normed:
        h = h / area
        yerr = yerr / area
        area = 1.

    if not 'color' in kwargs:
        kwargs['color'] = 'black'

    if not 'fmt' in kwargs:
        kwargs['fmt'] = 'o'

    plt.errorbar(center, h, xerr=xerr, yerr=yerr, **kwargs)

    return center, (yerr[0], h, yerr[1]), area

def make_split(ratio):
    import matplotlib.pyplot as plt
    from matplotlib.gridspec import GridSpec
    from matplotlib.ticker import MaxNLocator
    cax = plt.gca()
    box = cax.get_position()
    xmin, ymin = box.xmin, box.ymin
    xmax, ymax = box.xmax, box.ymax
    gs = GridSpec(2, 1, height_ratios=[ratio, 1 - ratio], left=xmin, right=xmax, bottom=ymin, top=ymax)
    gs.update(hspace=0.12)
    #gs.update(hspace=0.0)

    ax = plt.subplot(gs[0])
    plt.setp(ax.get_xticklabels(), visible=False)
    bx = plt.subplot(gs[1], sharex=ax)

    return ax, bx

def plot_pull(data, func):

    import numpy as np
    import matplotlib.pyplot as plt
    from matplotlib.ticker import MaxNLocator

    ax, bx = make_split(0.8)

    plt.sca(ax)

    x, y, norm = histpoints(data)

    lower, upper = ax.get_xlim()

    xs = np.linspace(lower, upper, 200)
    plt.plot(xs, norm * func(xs), 'b-')

    #plt.gca().yaxis.set_major_locator(MaxNLocator(prune='lower'))

    plt.sca(bx)

    resid = y[1] - norm * func(x)
    err = np.zeros_like(resid)
    err[resid >= 0] = y[0][resid >= 0]
    err[resid < 0] = y[2][resid < 0]

    pull = resid / err

    plt.errorbar(x, pull, yerr=1, color='k', capsize=0, markersize=0, fmt='o', lw=2)
    plt.ylim(-3, 3)
    plt.axhline(0, color='k', alpha=0.5)

    plt.sca(ax)

    return ax, bx

<|endoftext|>


<commit_before># api/views/mark.py

from ..core.setup import Resource

class Mark(Resource):
    def get(self, todo_id):
        return {todo_id: "test"}

class Marks(Resource):
    def get(self, todo_id):
        return {todo_id: "test"}
<commit_msg>pep8<commit_after># api/views/mark.py

from ..core.setup import Resource

class Mark(Resource):
    def get(self, todo_id):
        return {todo_id: "test"}

class Marks(Resource):
    def get(self, todo_id):
        return {todo_id: "test"}
<|endoftext|>


<commit_before>"""
Command-line entry to AnchorHub, main method is here.
"""

import anchorhub.cmdparse as cmdparse
import anchorhub.messages as messages
import anchorhub.normalization.normalize_opts as normalize_opts
import anchorhub.validation.validate_opts as validate_opts
import anchorhub.validation.validate_files as validate_files
from anchorhub.util.getfiles import get_files
from anchorhub.builtin.github.collector import make_github_markdown_collector

def main(argv=None):
    """
    Main entry method for AnchorHub. Takes in command-line arguments,
    finds files to parse within the specified input directory, and outputs
    parsed files to the specified output directory.

    :param argv: a list of string command line arguments
    """
    # Get command line arguments, validate them, and normalize them
    opts = cmdparse.parse_args(argv)
    assert validate_opts.validate(opts)
    opts = normalize_opts.normalize(opts)

    # Update client: print input and output directories
    messages.print_directories(opts)

    file_paths = get_files(opts.abs_input, opts.extensions,
                           exclude=[opts.abs_output])
    assert validate_files.validate(file_paths, opts)

    # Update client: print files that will be parsed
    messages.print_files(opts, file_paths)

    # For now, only using default GitHub Markdown for parsing
    # Collect tag/anchor combinations
    collector = make_github_markdown_collector(opts)
    anchors, duplicate_tags = collector.collect(file_paths)

if __name__ == '__main__':
    main()<commit_msg>Use GitHub Writer object in main.py<commit_after>"""
Command-line entry to AnchorHub, main method is here.
"""

import anchorhub.cmdparse as cmdparse
import anchorhub.messages as messages
import anchorhub.normalization.normalize_opts as normalize_opts
import anchorhub.validation.validate_opts as validate_opts
import anchorhub.validation.validate_files as validate_files
from anchorhub.util.getfiles import get_files
from anchorhub.builtin.github.collector import make_github_markdown_collector
from anchorhub.builtin.github.writer import make_github_markdown_writer

def main(argv=None):
    """
    Main entry method for AnchorHub. Takes in command-line arguments,
    finds files to parse within the specified input directory, and outputs
    parsed files to the specified output directory.

    :param argv: a list of string command line arguments
    """
    # Get command line arguments, validate them, and normalize them
    opts = cmdparse.parse_args(argv)
    assert validate_opts.validate(opts)
    opts = normalize_opts.normalize(opts)

    # Update client: print input and output directories
    messages.print_directories(opts)

    file_paths = get_files(opts.abs_input, opts.extensions,
                           exclude=[opts.abs_output])
    assert validate_files.validate(file_paths, opts)

    # Update client: print files that will be parsed
    messages.print_files(opts, file_paths)

    # For now, only using default GitHub Markdown for parsing
    # Collect tag/anchor combinations
    collector = make_github_markdown_collector(opts)
    anchors, duplicate_tags = collector.collect(file_paths)

    # Write
    writer = make_github_markdown_writer(opts)
    writer.write(file_paths, anchors, opts)

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>__version__ = '1.0.0'
<commit_msg>1.1.1<commit_after>__version__ = '1.1.1'
<|endoftext|>


<commit_before>import json
import urllib2
import datetime
import time
import collections
TrafficSample = collections.namedtuple('TrafficSample', ['segment_id', 'direction', 'road_status', 'update_time', 'aa', 'alert_counts', 'ac', 'alert_text' ])

ROADS = {
    1: "EDSA",
    2: "Q.AVE",
    3: "ESPANA",
    4: "C5",
    5: "ROXAS BLVD",
    6: "SLEX",
    7: "COMMONWEALTH",
    8: "ORTIGAS",
    9: "MARCOS HIWAY"
}
ROAD_STATUSES = {
    0: "NO INFO",
    1: "LIGHT",
    2: "LIGHT-MED",
    3: "MEDIUM",
    4: "MEDIUM-HEAVY",
    5: "HEAVY"
}
NODE_TYPES = {
    0: 'TERMINATION',
    1: 'ROAD',
    2: 'INTERSECTION'
}
TRAFFIC_ENDPOINT = 'http://mmdatraffic.interaksyon.com/data.traffic.status.php'
ADVISORIES_ENDPOINT = 'http://mmdatraffic.interaksyon.com/data.traffic.advisories.php'

response = urllib2.urlopen(TRAFFIC_ENDPOINT)
traffic_points = json.load(response)

for point in traffic_points:
    node_info = point[0]
    node_north = point[1]
    node_south = point[2]
    (road_id, node_id, intersecting_node_id, node_type_arr, is_service_node, related_node_id, is_major_intersection) = node_info
    (road_status, update_time, aa, alert_counts, ac) = node_north
    n_sample = TrafficSample(node_id, 'N', road_status, update_time, aa, alert_counts, ac, '')
    (road_status, update_time, aa, alert_counts, ac) = node_south
    s_sample = TrafficSample(node_id, 'S', road_status, update_time, aa, alert_counts, ac, '')

response = urllib2.urlopen(ADVISORIES_ENDPOINT)
advisories_list = json.load(response)
advisories = {}
# SegmentNames[142]
advisories = {}
for advisory in advisories_list:
    segment_id = advisory[1]
    info = advisory[2]
    print info

sample = traffic_points[2]
node_info = sample[0]
node_north = sample[1]
node_south = sample[2]

#is_major_intersection = Intersection between two roads
(road_id, node_id, intersecting_node_id, node_type_arr, is_service_node, related_node_id, is_major_intersection) = node_info
(road_status, update_time, aa, alert_counts, ac) = node_north
(node_type, node_type_idx, node_type_com) = node_type_arr

a = time.strptime(update_time, "%Y%m%d%H%M%S")
<commit_msg>Trivial<commit_after>import json
import urllib2
import datetime
import time
import collections
TrafficSample = collections.namedtuple('TrafficSample', ['segment_id', 'direction', 'road_status', 'update_time', 'aa', 'alert_counts', 'ac', 'alert_text' ])

#Comment

ROADS = {
    1: "EDSA",
    2: "Q.AVE",
    3: "ESPANA",
    4: "C5",
    5: "ROXAS BLVD",
    6: "SLEX",
    7: "COMMONWEALTH",
    8: "ORTIGAS",
    9: "MARCOS HIWAY"
}
ROAD_STATUSES = {
    0: "NO INFO",
    1: "LIGHT",
    2: "LIGHT-MED",
    3: "MEDIUM",
    4: "MEDIUM-HEAVY",
    5: "HEAVY"
}
NODE_TYPES = {
    0: 'TERMINATION',
    1: 'ROAD',
    2: 'INTERSECTION'
}
TRAFFIC_ENDPOINT = 'http://mmdatraffic.interaksyon.com/data.traffic.status.php'
ADVISORIES_ENDPOINT = 'http://mmdatraffic.interaksyon.com/data.traffic.advisories.php'

response = urllib2.urlopen(TRAFFIC_ENDPOINT)
traffic_points = json.load(response)

for point in traffic_points:
    node_info = point[0]
    node_north = point[1]
    node_south = point[2]
    (road_id, node_id, intersecting_node_id, node_type_arr, is_service_node, related_node_id, is_major_intersection) = node_info
    (road_status, update_time, aa, alert_counts, ac) = node_north
    n_sample = TrafficSample(node_id, 'N', road_status, update_time, aa, alert_counts, ac, '')
    (road_status, update_time, aa, alert_counts, ac) = node_south
    s_sample = TrafficSample(node_id, 'S', road_status, update_time, aa, alert_counts, ac, '')

response = urllib2.urlopen(ADVISORIES_ENDPOINT)
advisories_list = json.load(response)
advisories = {}
# SegmentNames[142]
advisories = {}
for advisory in advisories_list:
    segment_id = advisory[1]
    info = advisory[2]
    print info

sample = traffic_points[2]
node_info = sample[0]
node_north = sample[1]
node_south = sample[2]

#is_major_intersection = Intersection between two roads
(road_id, node_id, intersecting_node_id, node_type_arr, is_service_node, related_node_id, is_major_intersection) = node_info
(road_status, update_time, aa, alert_counts, ac) = node_north
(node_type, node_type_idx, node_type_com) = node_type_arr

a = time.strptime(update_time, "%Y%m%d%H%M%S")
<|endoftext|>


<commit_before>from flask_restful.reqparse import Argument
from flask_restful.reqparse import RequestParser as OriginalRequestParser

class RequestParser(OriginalRequestParser):

    def __init__(self, *args, **kargs):
        arguments = kargs.pop('arguments', None)
        select = kargs.pop('select', None)
        update = kargs.pop('update', None)
        remove = kargs.pop('remove', None)
        super(RequestParser, self).__init__(*args, **kargs)
        if arguments:
            if select:  # add just selected arguments
                arguments = dict((k, arguments[k]) for k in select)
            self.add_arguments(arguments)
        if update:
            self.update_arguments(update)
        if remove:
            self.remove_arguments(remove)

    def add_arguments(self, arguments):
        '''Add arguments to be parsed. Accepts either a dictionary of arguments
        or a list of `Argument`.'''
        if isinstance(arguments, dict):  # {'argument': {'required': ...}, }
            arguments = [Argument(*i) for i in arguments.items()]
        for argument in arguments:
            self.add_argument(argument)

    def update_arguments(self, dictionary):
        ''' Update the arguments matching the given dictionary keys. '''
        for arg in self.args:
            if arg.name in dictionary.keys():
                setattr(arg, arg.name, dictionary[arg.name])

    def remove_arguments(self, names):
        ''' Remove the arguments matching the given names. '''
        for name in names:
            self.remove_argument(name)

    def copy(self, update=None, remove=None):
        ''' Creates a copy with the same set of arguments. '''
        parsercopy = super(RequestParser, self).copy()
        # parsercopy = flask_restful.reqparse.RequestParser.copy(self)
        if update:
            parsercopy.update_arguments(update)
        if remove:
            parsercopy.remove_arguments(remove)
        return parsercopy
<commit_msg>Extending flask_restful.reqparse module.<commit_after>from flask_restful.reqparse import Argument
from flask_restful.reqparse import RequestParser as OriginalRequestParser

class RequestParser(OriginalRequestParser):

    def __init__(self, *args, **kargs):
        arguments = kargs.pop('arguments', None)
        select = kargs.pop('select', None)
        update = kargs.pop('update', None)
        remove = kargs.pop('remove', None)
        super(RequestParser, self).__init__(*args, **kargs)
        if arguments:
            if select:  # add just selected arguments
                arguments = dict((k, arguments[k]) for k in select)
            self.add_arguments(arguments)
        if update:
            self.update_arguments(update)
        if remove:
            self.remove_arguments(remove)

    def add_arguments(self, arguments):
        '''Add arguments to be parsed. Accepts either a dictionary of arguments
        or a list of `Argument`.'''
        if isinstance(arguments, dict):  # {'argument': {'required': ...}, }
            arguments = [Argument(*i) for i in arguments.items()]
        for argument in arguments:
            self.add_argument(argument)

    def update_arguments(self, dictionary):
        ''' Update the arguments matching the given dictionary keys. '''
        for arg in self.args:
            if arg.name in dictionary.keys():
                setattr(arg, arg.name, dictionary[arg.name])

    def remove_arguments(self, names):
        ''' Remove the arguments matching the given names. '''
        for name in names:
            self.remove_argument(name)

    def copy(self, update=None, remove=None):
        ''' Creates a copy with the same set of arguments. '''
        parsercopy = super(RequestParser, self).copy()
        if update:
            parsercopy.update_arguments(update)
        if remove:
            parsercopy.remove_arguments(remove)
        return parsercopy
<|endoftext|>