<commit_before>#!/usr/bin/env python
# Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Provides data sources to use for benchmarking."""

import glob
import os
import os.path
import random
import re
import subprocess
import sys
import time
import ConfigParser

# See if a third_party library exists -- use it if so.
try:
  import third_party
except ImportError:
  pass

# relative
import util
import selectors

# Pick the most accurate timer for a platform. Stolen from timeit.py:
if sys.platform[:3] == 'win':
  DEFAULT_TIMER = time.clock
else:
  DEFAULT_TIMER = time.time

GLOBAL_DATA_CACHE = {}
INTERNAL_RE = re.compile('^0|\.pro[md]z*\.|\.corp|\.bor|\.hot$|internal|dmz|\._[ut][dc]p\.|intra|\.\w$|\.\w{5,}$')
# ^.*[\w-]+\.[\w-]+\.[\w-]+\.[a-zA-Z]+\.$|^[\w-]+\.[\w-]{3,}\.[a-zA-Z]+\.$
FQDN_RE = re.compile('^.*\..*\..*\..*\.$|^.*\.[\w-]*\.\w{3,4}\.$|^[\w-]+\.[\w-]{4,}\.\w+\.')

IP_RE = re.compile('^[0-9.]+$')
DEFAULT_CONFIG_PATH = "config/data_sources.cfg"
MAX_NON_UNIQUE_RECORD_COUNT = 500000
MAX_FILE_MTIME_AGE_DAYS = 45
MIN_FILE_SIZE = 10000
MIN_RECOMMENDED_RECORD_COUNT = 200
MAX_FQDN_SYNTHESIZE_PERCENT = 4

class DataSources(object):
  def __init__(self, config_path=DEFAULT_CONFIG_PATH, status_callback=None):
    global GLOBAL_DATA_CACHE
    self.source_cache = GLOBAL_DATA_CACHE
    self.source_config = {}
    self.status_callback = status_callback
    self._LoadConfigFromPath(config_path)

  def msg(self, msg, **kwargs):
    if self.status_callback:
      self.status_callback(msg, **kwargs)
    else:
      print '- %s' % msg

  def _LoadConfigFromPath(self, path):
    conf_file = util.FindDataFile('config/data_sources.cfg')
    config = ConfigParser.ConfigParser()
    config.read(conf_file)
    for section in config.sections():
      if section not in self.source_config:
        self.source_config[section] = {
          'name': None,
          'search_paths': set(),
          # Store whether or not this data source contains personal data
          'full_hostnames': True,
          'include_duplicates': False,
          'max_mtime_days': MAX_FILE_MTIME_AGE_DAYS
        }

      for (key, value) in config.items(section):
        if key == 'name':
          self.source_config[section]['name'] = value
        elif key == 'full_hostnames' and int(value) == 0:
          self.source_config[section]['full_hostnames'] = False
        elif key == 'max_mtime_days':
          self.source_config[section]['max_mtime_days'] = int(value)
        elif key == 'include_duplicates':
          self.source_config[section]['include_duplicates'] = bool(value)
        else:
          self.source_config[section]['search_paths'].add(value)

  def ListSourceTypes(self):
    """Get a list of all data sources we know about."""
    return sorted(self.source_config.keys())

  def ListSourcesWithDetails(self):
    """Get a list of all data sources found with total counts.

    Returns:
      List of tuples in form of (short_name, full_name, full_hosts, # of entries)
    """
    for source in self.ListSourceTypes():
      max_mtime = self.source_config[source]['max_mtime_days']
      self._GetHostsFromSource(source, min_file_size=MIN_FILE_SIZE,
                               max_mtime_age_days=max_mtime)

    details = []
    for source in self.source_cache:
      details.append((source,
                      self.source_config[source]['name'],
                      self.source_config[source]['full_hostnames'],
                      len(self.source_cache[source])))
    return sorted(details, key=lambda x:(x[2], x[3]), reverse=True)

  def ListSourceTitles(self):
    """Return a list of sources in title + count format."""
    titles = []
    for (source_type, name, full_hostnames, count) in self.ListSourcesWithDetails():
      titles.append("%s (%s)" % (name, count))
    return titles

  def ConvertSourceTitleToType(self, detail):
    """Convert a detail name to a source type."""
    for source_type in self.source_config:
      if detail.startswith(self.source_config[source_type]['name']):
        return source_type

  def GetBestSourceDetails(self):
    return self.ListSourcesWithDetails()[0]

  def GetNameForSource(self, source):
    if source in self.source_config:
      return self.source_config[source]['name']
    else:
      # Most likely a custom file path
      return source

  def GetCachedRecordCountForSource(self, source):
    return len(self.source_cache[source])

  def _CreateRecordsFromHostEntries(self, entries, include_duplicates=False):
    """Create records from hosts, removing duplicate entries and IP's

    Args:
      A list of test-data entries.

    Returns:
      A tuple of (filtered records, full_host_names (Boolean)
    """
    real_tld_re = re.compile('[a-z]{2,4}$')
    internal_re = re.compile('^[\d:\.]+$')
    last_entry = None

    records = []
    full_host_count = 0
    for entry in entries:
      if entry == last_entry and not include_duplicates:
        continue
      else:
        last_entry = entry

      if ' ' in entry:
        (record_type, host) = entry.split(' ')
      else:
        record_type = 'A'
        host = entry

      if not IP_RE.match(host) and not INTERNAL_RE.search(host):
        if not host.endswith('.'):
          # For a short string like this, simple addition is 54% faster than formatting
          host = host + '.'
        records.append((record_type, host))

        if FQDN_RE.match(host):
          full_host_count += 1

    if not len(records):
      raise ValueError("No records could be created from: %s" % entries)

    # Now that we've read everything, are we dealing with domains or full hostnames?
    full_host_percent = full_host_count / float(len(records)) * 100
    if full_host_percent < MAX_FQDN_SYNTHESIZE_PERCENT:
      full_host_names = True
    else:
      full_host_names = False
    return (records, full_host_names)

  def GetTestsFromSource(self, source, count=50, select_mode=None):
    """Parse records from source, and returnrequest tuples to use for testing.

    This is tricky because we support 3 types of input data:

    - List of domains
    - List of hosts
    - List of record_type + hosts
    """
    records = []

    # Convert entries into tuples, determine if we are using full hostnames
    full_host_count = 0
    www_host_count = 0
    include_duplicates = self.source_config[source]['include_duplicates']
    records = self._GetHostsFromSource(source)
    self.msg('Generating tests from %s (%s records, selecting %s %s)'
             % (self.GetNameForSource(source), len(records), count, select_mode))
    (records, are_records_fqdn) = self._CreateRecordsFromHostEntries(records,
                                                                     include_duplicates=include_duplicates)
    # First try to resolve whether to use weighted or random.
    if select_mode in ('weighted', 'automatic', None):
      # If we are in include_duplicates mode (cachemiss, cachehit, etc.), we have different rules.
      if include_duplicates:
        if count > len(records) :
          records = selectors.RandomSelect(records, count, include_duplicates=include_duplicates)
        else:
          records = selectors.ChunkSelect(records, count)
      elif len(records) != len(set(records)):
        if select_mode == 'weighted':
          self.msg('%s data contains duplicates, switching select_mode to random' % source)
        select_mode = 'random'
      else:
        select_mode = 'weighted'

    self.msg('Selecting %s out of %s sanitized records (%s mode).' % (count, len(records), select_mode))
    if select_mode == 'weighted':
      records = selectors.WeightedDistribution(records, count)
    elif select_mode == 'chunk':
      records = selectors.ChunkSelect(records, count)
    elif select_mode == 'random':
      records = selectors.RandomSelect(records, count)

    # For custom filenames
    if source not in self.source_config:
      self.source_config[source] = {}

    if are_records_fqdn:
      self.source_config[source]['full_hostnames'] = False
      self.msg('%s input appears to be predominantly domain names. Synthesizing FQDNs' % source)
      synthesized = []
      for (req_type, hostname) in records:
        if not FQDN_RE.match(hostname):
          hostname = self._GenerateRandomHostname(hostname)
        synthesized.append((req_type, hostname))
      return synthesized
    else:
      return records

  def _GenerateRandomHostname(self, domain):
    """Generate a random hostname f or a given domain."""
    oracle = random.randint(0, 100)
    if oracle < 70:
      return 'www.%s' % domain
    elif oracle < 95:
      return domain
    elif oracle < 98:
      return 'static.%s' % domain
    else:
      return 'cache-%s.%s' % (random.randint(0, 10), domain)

  def _GetHostsFromSource(self, source, min_file_size=None, max_mtime_age_days=None):
    """Get data for a particular source. This needs to be fast.

    We support 3 styles of files:

    * One-per line list in form of record-type: host
    * One-per line list of unique domains
    * Any form with URL's.

    The results of this function get cached.
    """
    if source in self.source_cache:
      return self.source_cache[source]
    filename = self._FindBestFileForSource(source, min_file_size=min_file_size,
                                           max_mtime_age_days=max_mtime_age_days)
    if not filename:
      return None

    size_mb = os.path.getsize(filename) / 1024.0 / 1024.0
    self.msg('Reading %s: %s (%0.1fMB)' % (self.GetNameForSource(source), filename, size_mb))
    start_clock = DEFAULT_TIMER()
    if filename.endswith('.pcap') or filename.endswith('.tcp'):
      hosts = self._ExtractHostsFromPcapFile(filename)
    else:
      hosts = self._ExtractHostsFromHistoryFile(filename)

    if not hosts:
      hosts = self._ReadDataFile(filename)
    duration = DEFAULT_TIMER() - start_clock
    if duration > 5:
      self.msg('%s data took %1.1fs to read!' % (self.GetNameForSource(source), duration))
    self.source_cache[source] = hosts
    return hosts

  def _ExtractHostsFromHistoryFile(self, path):
    """Get a list of sanitized records from a history file containing URLs."""
    # This regexp is fairly general (no ip filtering), since we need speed more
    # than precision at this stage.
    parse_re = re.compile('https*://([\-\w]+\.[\-\w\.]+)')
    return parse_re.findall(open(path, 'rb').read())

  def _ExtractHostsFromPcapFile(self, path):
    """Get a list of requests out of a pcap file (requires tcpdump)"""
    self.msg("Extracting requests from pcap file using tcpdump")
    cmd = 'tcpdump -r %s -n port 53' % path
    pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout
    parse_re = re.compile(' ([A-Z]+)\? ([\-\w\.]+)')
    requests = []
    for line in pipe:
      if '?' not in line:
        continue
      match = parse_re.search(line)
      if match:
        requests.append(' '.join(match.groups()))
    return requests

  def _ReadDataFile(self, path):
    """Read a line-based datafile."""
    records = []
    domains_re = re.compile('^\w[\w\.-]+[a-zA-Z]$')
    requests_re = re.compile('^[A-Z]{1,4} \w[\w\.-]+\.$')
    for line in open(path).readlines():
      if domains_re.match(line) or requests_re.match(line):
        records.append(line.rstrip())
    return records

  def _GetSourceSearchPaths(self, source):
    """Get a list of possible search paths (globs) for a given source."""

    # This is likely a custom file path
    if source not in self.source_config:
      return [source]

    search_paths = []
    environment_re = re.compile('%(\w+)%')

    # First get through resolving environment variables
    for path in self.source_config[source]['search_paths']:
      env_vars = set(environment_re.findall(path))
      if env_vars:
        for variable in env_vars:
          env_var = os.getenv(variable, False)
          if env_var:
            path = path.replace('%%%s%%' % variable, env_var)
          else:
            path = None

      # If everything is good, replace all '/'  chars with the os path variable.
      if path:
        path = path.replace('/', os.sep)
        search_paths.append(path)

        # This moment of weirdness brought to you by Windows XP(tm). If we find
        # a Local or Roaming keyword in path, add the other forms to the search
        # path.
        if sys.platform[:3] == 'win':
          keywords = ('Local', 'Roaming')
          for keyword in keywords:
            if keyword in path:
              replacement = keywords[keywords.index(keyword)-1]
              search_paths.append(path.replace('\\%s' % keyword, '\\%s' % replacement))
              search_paths.append(path.replace('\\%s' % keyword, ''))

    return search_paths

  def _FindBestFileForSource(self, source, min_file_size=None,
                             max_mtime_age_days=None):
    """Find the best file (newest over X size) to use for a given source type.

    Args:
      source: source type

    Returns:
      A file path.
    """
    found = []
    for path in self._GetSourceSearchPaths(source):
      if not os.path.isabs(path):
        path = util.FindDataFile(path)

      for filename in glob.glob(path):
        if min_file_size and os.path.getsize(filename) < min_file_size:
          self.msg('Skipping %s (only %sb)' % (filename, os.path.getsize(filename)))
        else:
          found.append(filename)

    if found:
      newest = sorted(found, key=os.path.getmtime)[-1]
      age_days = (time.time() - os.path.getmtime(newest)) / 86400
      if max_mtime_age_days and age_days > max_mtime_age_days:
        self.msg('Skipping %s (%2.0fd old)' % (newest, age_days))
      else:
        return newest
    else:
      return None

if __name__ == '__main__':
  parser = DataSources()
  print parser.ListSourceTypes()
  print parser.ListSourcesWithDetails()
  best = parser.ListSourcesWithDetails()[0][0]
  print len(parser.GetRecordsFromSource(best))
<commit_msg>Add raise if we hit a select_mode bug<commit_after>#!/usr/bin/env python
# Copyright 2009 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Provides data sources to use for benchmarking."""

import glob
import os
import os.path
import random
import re
import subprocess
import sys
import time
import ConfigParser

# See if a third_party library exists -- use it if so.
try:
  import third_party
except ImportError:
  pass

# relative
import util
import selectors

# Pick the most accurate timer for a platform. Stolen from timeit.py:
if sys.platform[:3] == 'win':
  DEFAULT_TIMER = time.clock
else:
  DEFAULT_TIMER = time.time

GLOBAL_DATA_CACHE = {}
INTERNAL_RE = re.compile('^0|\.pro[md]z*\.|\.corp|\.bor|\.hot$|internal|dmz|\._[ut][dc]p\.|intra|\.\w$|\.\w{5,}$')
# ^.*[\w-]+\.[\w-]+\.[\w-]+\.[a-zA-Z]+\.$|^[\w-]+\.[\w-]{3,}\.[a-zA-Z]+\.$
FQDN_RE = re.compile('^.*\..*\..*\..*\.$|^.*\.[\w-]*\.\w{3,4}\.$|^[\w-]+\.[\w-]{4,}\.\w+\.')

IP_RE = re.compile('^[0-9.]+$')
DEFAULT_CONFIG_PATH = "config/data_sources.cfg"
MAX_NON_UNIQUE_RECORD_COUNT = 500000
MAX_FILE_MTIME_AGE_DAYS = 45
MIN_FILE_SIZE = 10000
MIN_RECOMMENDED_RECORD_COUNT = 200
MAX_FQDN_SYNTHESIZE_PERCENT = 4

class DataSources(object):
  def __init__(self, config_path=DEFAULT_CONFIG_PATH, status_callback=None):
    global GLOBAL_DATA_CACHE
    self.source_cache = GLOBAL_DATA_CACHE
    self.source_config = {}
    self.status_callback = status_callback
    self._LoadConfigFromPath(config_path)

  def msg(self, msg, **kwargs):
    if self.status_callback:
      self.status_callback(msg, **kwargs)
    else:
      print '- %s' % msg

  def _LoadConfigFromPath(self, path):
    conf_file = util.FindDataFile('config/data_sources.cfg')
    config = ConfigParser.ConfigParser()
    config.read(conf_file)
    for section in config.sections():
      if section not in self.source_config:
        self.source_config[section] = {
          'name': None,
          'search_paths': set(),
          # Store whether or not this data source contains personal data
          'full_hostnames': True,
          'include_duplicates': False,
          'max_mtime_days': MAX_FILE_MTIME_AGE_DAYS
        }

      for (key, value) in config.items(section):
        if key == 'name':
          self.source_config[section]['name'] = value
        elif key == 'full_hostnames' and int(value) == 0:
          self.source_config[section]['full_hostnames'] = False
        elif key == 'max_mtime_days':
          self.source_config[section]['max_mtime_days'] = int(value)
        elif key == 'include_duplicates':
          self.source_config[section]['include_duplicates'] = bool(value)
        else:
          self.source_config[section]['search_paths'].add(value)

  def ListSourceTypes(self):
    """Get a list of all data sources we know about."""
    return sorted(self.source_config.keys())

  def ListSourcesWithDetails(self):
    """Get a list of all data sources found with total counts.

    Returns:
      List of tuples in form of (short_name, full_name, full_hosts, # of entries)
    """
    for source in self.ListSourceTypes():
      max_mtime = self.source_config[source]['max_mtime_days']
      self._GetHostsFromSource(source, min_file_size=MIN_FILE_SIZE,
                               max_mtime_age_days=max_mtime)

    details = []
    for source in self.source_cache:
      details.append((source,
                      self.source_config[source]['name'],
                      self.source_config[source]['full_hostnames'],
                      len(self.source_cache[source])))
    return sorted(details, key=lambda x:(x[2], x[3]), reverse=True)

  def ListSourceTitles(self):
    """Return a list of sources in title + count format."""
    titles = []
    for (source_type, name, full_hostnames, count) in self.ListSourcesWithDetails():
      titles.append("%s (%s)" % (name, count))
    return titles

  def ConvertSourceTitleToType(self, detail):
    """Convert a detail name to a source type."""
    for source_type in self.source_config:
      if detail.startswith(self.source_config[source_type]['name']):
        return source_type

  def GetBestSourceDetails(self):
    return self.ListSourcesWithDetails()[0]

  def GetNameForSource(self, source):
    if source in self.source_config:
      return self.source_config[source]['name']
    else:
      # Most likely a custom file path
      return source

  def GetCachedRecordCountForSource(self, source):
    return len(self.source_cache[source])

  def _CreateRecordsFromHostEntries(self, entries, include_duplicates=False):
    """Create records from hosts, removing duplicate entries and IP's

    Args:
      A list of test-data entries.

    Returns:
      A tuple of (filtered records, full_host_names (Boolean)
    """
    real_tld_re = re.compile('[a-z]{2,4}$')
    internal_re = re.compile('^[\d:\.]+$')
    last_entry = None

    records = []
    full_host_count = 0
    for entry in entries:
      if entry == last_entry and not include_duplicates:
        continue
      else:
        last_entry = entry

      if ' ' in entry:
        (record_type, host) = entry.split(' ')
      else:
        record_type = 'A'
        host = entry

      if not IP_RE.match(host) and not INTERNAL_RE.search(host):
        if not host.endswith('.'):
          # For a short string like this, simple addition is 54% faster than formatting
          host = host + '.'
        records.append((record_type, host))

        if FQDN_RE.match(host):
          full_host_count += 1

    if not len(records):
      raise ValueError("No records could be created from: %s" % entries)

    # Now that we've read everything, are we dealing with domains or full hostnames?
    full_host_percent = full_host_count / float(len(records)) * 100
    if full_host_percent < MAX_FQDN_SYNTHESIZE_PERCENT:
      full_host_names = True
    else:
      full_host_names = False
    return (records, full_host_names)

  def GetTestsFromSource(self, source, count=50, select_mode=None):
    """Parse records from source, and returnrequest tuples to use for testing.

    This is tricky because we support 3 types of input data:

    - List of domains
    - List of hosts
    - List of record_type + hosts
    """
    records = []

    # Convert entries into tuples, determine if we are using full hostnames
    full_host_count = 0
    www_host_count = 0
    include_duplicates = self.source_config[source]['include_duplicates']
    records = self._GetHostsFromSource(source)
    self.msg('Generating tests from %s (%s records, selecting %s %s)'
             % (self.GetNameForSource(source), len(records), count, select_mode))
    (records, are_records_fqdn) = self._CreateRecordsFromHostEntries(records,
                                                                     include_duplicates=include_duplicates)
    # First try to resolve whether to use weighted or random.
    if select_mode in ('weighted', 'automatic', None):
      # If we are in include_duplicates mode (cachemiss, cachehit, etc.), we have different rules.
      if include_duplicates:
        if count > len(records) :
          records = selectors.RandomSelect(records, count, include_duplicates=include_duplicates)
        else:
          records = selectors.ChunkSelect(records, count)
      elif len(records) != len(set(records)):
        if select_mode == 'weighted':
          self.msg('%s data contains duplicates, switching select_mode to random' % source)
        select_mode = 'random'
      else:
        select_mode = 'weighted'

    self.msg('Selecting %s out of %s sanitized records (%s mode).' % (count, len(records), select_mode))
    if select_mode == 'weighted':
      records = selectors.WeightedDistribution(records, count)
    elif select_mode == 'chunk':
      records = selectors.ChunkSelect(records, count)
    elif select_mode == 'random':
      records = selectors.RandomSelect(records, count)
    else:
      raise ValueError("No such final selection mode: %s" % select_mode)

    # For custom filenames
    if source not in self.source_config:
      self.source_config[source] = {}

    if are_records_fqdn:
      self.source_config[source]['full_hostnames'] = False
      self.msg('%s input appears to be predominantly domain names. Synthesizing FQDNs' % source)
      synthesized = []
      for (req_type, hostname) in records:
        if not FQDN_RE.match(hostname):
          hostname = self._GenerateRandomHostname(hostname)
        synthesized.append((req_type, hostname))
      return synthesized
    else:
      return records

  def _GenerateRandomHostname(self, domain):
    """Generate a random hostname f or a given domain."""
    oracle = random.randint(0, 100)
    if oracle < 70:
      return 'www.%s' % domain
    elif oracle < 95:
      return domain
    elif oracle < 98:
      return 'static.%s' % domain
    else:
      return 'cache-%s.%s' % (random.randint(0, 10), domain)

  def _GetHostsFromSource(self, source, min_file_size=None, max_mtime_age_days=None):
    """Get data for a particular source. This needs to be fast.

    We support 3 styles of files:

    * One-per line list in form of record-type: host
    * One-per line list of unique domains
    * Any form with URL's.

    The results of this function get cached.
    """
    if source in self.source_cache:
      return self.source_cache[source]
    filename = self._FindBestFileForSource(source, min_file_size=min_file_size,
                                           max_mtime_age_days=max_mtime_age_days)
    if not filename:
      return None

    size_mb = os.path.getsize(filename) / 1024.0 / 1024.0
    self.msg('Reading %s: %s (%0.1fMB)' % (self.GetNameForSource(source), filename, size_mb))
    start_clock = DEFAULT_TIMER()
    if filename.endswith('.pcap') or filename.endswith('.tcp'):
      hosts = self._ExtractHostsFromPcapFile(filename)
    else:
      hosts = self._ExtractHostsFromHistoryFile(filename)

    if not hosts:
      hosts = self._ReadDataFile(filename)
    duration = DEFAULT_TIMER() - start_clock
    if duration > 5:
      self.msg('%s data took %1.1fs to read!' % (self.GetNameForSource(source), duration))
    self.source_cache[source] = hosts
    return hosts

  def _ExtractHostsFromHistoryFile(self, path):
    """Get a list of sanitized records from a history file containing URLs."""
    # This regexp is fairly general (no ip filtering), since we need speed more
    # than precision at this stage.
    parse_re = re.compile('https*://([\-\w]+\.[\-\w\.]+)')
    return parse_re.findall(open(path, 'rb').read())

  def _ExtractHostsFromPcapFile(self, path):
    """Get a list of requests out of a pcap file (requires tcpdump)"""
    self.msg("Extracting requests from pcap file using tcpdump")
    cmd = 'tcpdump -r %s -n port 53' % path
    pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout
    parse_re = re.compile(' ([A-Z]+)\? ([\-\w\.]+)')
    requests = []
    for line in pipe:
      if '?' not in line:
        continue
      match = parse_re.search(line)
      if match:
        requests.append(' '.join(match.groups()))
    return requests

  def _ReadDataFile(self, path):
    """Read a line-based datafile."""
    records = []
    domains_re = re.compile('^\w[\w\.-]+[a-zA-Z]$')
    requests_re = re.compile('^[A-Z]{1,4} \w[\w\.-]+\.$')
    for line in open(path).readlines():
      if domains_re.match(line) or requests_re.match(line):
        records.append(line.rstrip())
    return records

  def _GetSourceSearchPaths(self, source):
    """Get a list of possible search paths (globs) for a given source."""

    # This is likely a custom file path
    if source not in self.source_config:
      return [source]

    search_paths = []
    environment_re = re.compile('%(\w+)%')

    # First get through resolving environment variables
    for path in self.source_config[source]['search_paths']:
      env_vars = set(environment_re.findall(path))
      if env_vars:
        for variable in env_vars:
          env_var = os.getenv(variable, False)
          if env_var:
            path = path.replace('%%%s%%' % variable, env_var)
          else:
            path = None

      # If everything is good, replace all '/'  chars with the os path variable.
      if path:
        path = path.replace('/', os.sep)
        search_paths.append(path)

        # This moment of weirdness brought to you by Windows XP(tm). If we find
        # a Local or Roaming keyword in path, add the other forms to the search
        # path.
        if sys.platform[:3] == 'win':
          keywords = ('Local', 'Roaming')
          for keyword in keywords:
            if keyword in path:
              replacement = keywords[keywords.index(keyword)-1]
              search_paths.append(path.replace('\\%s' % keyword, '\\%s' % replacement))
              search_paths.append(path.replace('\\%s' % keyword, ''))

    return search_paths

  def _FindBestFileForSource(self, source, min_file_size=None,
                             max_mtime_age_days=None):
    """Find the best file (newest over X size) to use for a given source type.

    Args:
      source: source type

    Returns:
      A file path.
    """
    found = []
    for path in self._GetSourceSearchPaths(source):
      if not os.path.isabs(path):
        path = util.FindDataFile(path)

      for filename in glob.glob(path):
        if min_file_size and os.path.getsize(filename) < min_file_size:
          self.msg('Skipping %s (only %sb)' % (filename, os.path.getsize(filename)))
        else:
          found.append(filename)

    if found:
      newest = sorted(found, key=os.path.getmtime)[-1]
      age_days = (time.time() - os.path.getmtime(newest)) / 86400
      if max_mtime_age_days and age_days > max_mtime_age_days:
        self.msg('Skipping %s (%2.0fd old)' % (newest, age_days))
      else:
        return newest
    else:
      return None

if __name__ == '__main__':
  parser = DataSources()
  print parser.ListSourceTypes()
  print parser.ListSourcesWithDetails()
  best = parser.ListSourcesWithDetails()[0][0]
  print len(parser.GetRecordsFromSource(best))
<|endoftext|>


<commit_before>import community
import networkx as nx
import pandas as pd
from random import shuffle

class GraphPrep:
    def __init__(self, path):
        '''Constructor'''
        edges = pd.read_csv(path, header=0)
        self.l = edges.columns.values.tolist()
        self.l.remove('source')
        self.l.remove('target')
        self.G = nx.from_pandas_dataframe(edges, 'source', 'target', self.l, create_using=nx.DiGraph())

    def swap_edges(self):
        # Random small world
        # Random mix of edges
        # Reverso of edges
        self.G = nx.double_edge_swap(self.G, 50, 150)
        return self

    def shuffle_target(self):
        edges = self.G.edges(data=True)

        for e in edges:
            self.G.remove_edge(e[0], e[1])

        target = [e[1] for e in edges]
        shuffle(target)

        for e1, e2 in zip(edges, target):
            self.G.add_edge(e1[0], e2, e1[2])

        edges = self.G.edges(data=True)

        return self

    def shuffle_source(self):
        edges = self.G.edges(data=True)

        for e in edges:
            self.G.remove_edge(e[0], e[1])

        target = [e[0] for e in edges]
        shuffle(target)

        for e1, e2 in zip(edges, target):
            self.G.add_edge(e2, e1[1], e1[2])

        return self

    def detect_communities(self):
        partition = community.best_partition(self.G)
        for n in partition:
            nx.set_node_attributes(self.G, 'community', {n: partition[n]})

        self.l.append("community")
        return self

    def save_to_pickle(self, path):
        nx.write_gpickle(self.G, path, protocol=2)
        return self

    def save_edges_to_csv(self, path):
        fh = open(path, 'wb')
        nx.write_edgelist(self.G, fh, delimiter=',', data=self.l)
        return self

    def save_nodes_to_csv(self, path):
        fh = open(path, 'wb')
        for n in self.G.nodes(data=True):
            line = []
            line.append(str(n[0]))
            for x in self.l:
                if x in n[1]:
                    line.append(str(n[1][x]))
            fh.write(",".join(line))
            fh.write("\n")
        return self

    def deg_graph(self):
        return self.G

    def to_undriected(self):
        G_tmp = self.G.to_undirected()  # copy
        for (u, v) in self.G.edges():
            if not self.G.has_edge(v, u):
                G_tmp.remove_edge(u, v)
            else:
                avr = (self.G.get_edge_data(u, v)["weight"] + self.G.get_edge_data(v, u)["weight"]) / 2
                G_tmp.add_edge(u, v, weight=avr)
        self.G = G_tmp
        return self

    def remove_self_loop(self):
        self.G.remove_edges_from(self.G.selfloop_edges())
        return self

if __name__ == '__main__':
    GraphPrep(
        "/Users/danielkershaw/PycharmProjects/DiffusionSimulation/data/Twitter_GEO_network.csv"). \
        to_undriected(). \
        remove_self_loop(). \
        detect_communities(). \
        save_to_pickle("./../data/go-pickle-2.gpickle")
    # save_to_pickle("../data/Twitter_GEO_network.pickle")
<commit_msg>Ability to remove self-loop edges.<commit_after>from random import shuffle

import community
import networkx as nx
import pandas as pd

class GraphPrep:
    def __init__(self, path):
        '''Constructor'''
        edges = pd.read_csv(path, header=0)
        self.l = edges.columns.values.tolist()
        self.l.remove('source')
        self.l.remove('target')
        self.G = nx.from_pandas_dataframe(edges, 'source', 'target', self.l, create_using=nx.DiGraph())

    def swap_edges(self):
        # Random small world
        # Random mix of edges
        # Reverso of edges
        self.G = nx.double_edge_swap(self.G, 50, 150)
        return self

    def shuffle_target(self):
        edges = self.G.edges(data=True)

        for e in edges:
            self.G.remove_edge(e[0], e[1])

        target = [e[1] for e in edges]
        shuffle(target)

        for e1, e2 in zip(edges, target):
            self.G.add_edge(e1[0], e2, e1[2])

        edges = self.G.edges(data=True)

        return self

    def shuffle_source(self):
        edges = self.G.edges(data=True)

        for e in edges:
            self.G.remove_edge(e[0], e[1])

        target = [e[0] for e in edges]
        shuffle(target)

        for e1, e2 in zip(edges, target):
            self.G.add_edge(e2, e1[1], e1[2])

        return self

    def detect_communities(self):
        partition = community.best_partition(self.G)
        for n in partition:
            nx.set_node_attributes(self.G, 'community', {n: partition[n]})

        self.l.append("community")
        return self

    def save_to_pickle(self, path):
        nx.write_gpickle(self.G, path, protocol=2)
        return self

    def save_edges_to_csv(self, path):
        fh = open(path, 'wb')
        nx.write_edgelist(self.G, fh, delimiter=',', data=self.l)
        return self

    def save_nodes_to_csv(self, path):
        fh = open(path, 'wb')
        for n in self.G.nodes(data=True):
            line = []
            line.append(str(n[0]))
            for x in self.l:
                if x in n[1]:
                    line.append(str(n[1][x]))
            fh.write(",".join(line))
            fh.write("\n")
        return self

    def deg_graph(self):
        return self.G

    def to_undriected(self):
        G_tmp = self.G.to_undirected()  # copy
        for (u, v) in self.G.edges():
            if not self.G.has_edge(v, u):
                G_tmp.remove_edge(u, v)
            else:
                avr = (self.G.get_edge_data(u, v)["weight"] + self.G.get_edge_data(v, u)["weight"]) / 2
                G_tmp.add_edge(u, v, weight=avr)
        self.G = G_tmp
        return self

    def remove_self_loop(self):
        self.G.remove_edges_from(self.G.selfloop_edges())
        return self

if __name__ == '__main__':
    GraphPrep(
        "/Users/kershad1/PycharmProjects/DiffusionSimulation/data/user_reddit_comment_network_2014_significant"). \
        to_undriected(). \
        remove_self_loop(). \
        detect_communities(). \
        save_to_pickle("./../data/pickle/reddit-comment-significant-2014.gpickle")

    GraphPrep(
        "/Users/kershad1/PycharmProjects/DiffusionSimulation/data/user_reddit_comment_network_2014_significant"). \
        shuffle_target(). \
        save_edges_to_csv("./../data/shuffel/reddit-comment-significant-2014.csv")
    # save_to_pickle("../data/Twitter_GEO_network.pickle")
<|endoftext|>


<commit_before>import uuid
import contextlib

from tvtk.api import tvtk

from simphony.cuds.abstractmesh import ABCMesh
from simphony.cuds.mesh import Point, Edge, Face, Cell
from simphony.core.data_container import DataContainer
from simphony_mayavi.core.api import (
    CubaData, CellCollection, supported_cuba, mergedocs,
    VTKEDGETYPES, EDGE2VTKCELL, FACE2VTKCELL, VTKFACETYPES,
    VTKCELLTYPES, CELL2VTKCELL)

@mergedocs(ABCMesh)
class VTKMesh(ABCMesh):

    def __init__(self, name, data=None, data_set=None):
        self.name = name
        self._data = DataContainer() if data is None else DataContainer(data)
        #: The mapping from uid to point index
        self.point2index = {}
        #: The reverse mapping from index to point uid
        self.index2point = {}
        #: The mapping from uid to bond index
        self.element2index = {}
        #: The reverse mapping from index to bond uid
        self.index2element = {}

        # Setup the data_set
        if data_set is None:
            points = tvtk.Points()
            # Need to initialise lines with empty so that we
            # do not get the shared CellArray
            data_set = tvtk.UnstructuredGrid(points=points)
        else:
            for index in xrange(data_set.number_of_points):
                uid = uuid.uuid4()
                self.point2index[uid] = index
                self.index2point[index] = uid
            for index in xrange(data_set.number_of_cells):
                uid = uuid.uuid4()
                self.element2index[uid] = index
                self.index2element[index] = uid

        #: The vtk.PolyData dataset
        self.data_set = data_set

        #: The currently supported and stored CUBA keywords.
        self.supported_cuba = supported_cuba()

        #: Easy access to the vtk PointData structure
        data = data_set.point_data
        if data.number_of_arrays == 0:
            size = data_set.number_of_points
        else:
            size = None
        self.point_data = CubaData(
            data, stored_cuba=self.supported_cuba, size=size)

        #: Easy access to the vtk CellData structure
        data = data_set.cell_data
        ncells = data_set.number_of_cells
        if data.number_of_arrays == 0 and ncells != 0:
            size = ncells
        else:
            size = None
        self.element_data = CubaData(
            data, stored_cuba=self.supported_cuba, size=size)

        # Elements cells
        self.elements = CellCollection(data_set.get_cells())

    @property
    def data(self):
        """ The container data
        """
        return DataContainer(self._data)

    @data.setter
    def data(self, value):
        self._data = DataContainer(value)

    # Point operations ####################################################

    def add_point(self, point):
        data_set = self.data_set
        points = data_set.points
        point2index = self.point2index
        with self._add_item(point, point2index) as item:
            index = points.insert_next_point(item.coordinates)
            point2index[item.uid] = index
            self.index2point[index] = item.uid
            self.point_data.append(item.data)
            return item.uid

    def get_point(self, uid):
        if not isinstance(uid, uuid.UUID):
            raise TypeError("{} is not a uuid".format(uid))
        index = int(self.point2index[uid])
        return Point(
            uid=uid,
            coordinates=self.data_set.points[index],
            data=self.point_data[index])

    def update_point(self, point):
        try:
            index = self.point2index[point.uid]
        except KeyError:
            message = "Point with {} does exist"
            raise ValueError(message.format(point.uid))
        # Need to cast to int https://github.com/enthought/mayavi/issues/173
        self.data_set.points[int(index)] = point.coordinates
        self.point_data[index] = point.data

    def iter_points(self, uids=None):
        if uids is None:
            for uid in self.point2index:
                yield self.get_point(uid)
        else:
            for uid in uids:
                yield self.get_point(uid)

    # Edge operations ########################################################

    def add_edge(self, edge):
        data_set = self.data_set
        element2index = self.element2index
        with self._add_item(edge, element2index) as item:
            point_ids = [self.point2index[uid] for uid in item.points]
            index = data_set.insert_next_cell(
                EDGE2VTKCELL[len(point_ids)], point_ids)
            element2index[item.uid] = index
            self.index2element[index] = item.uid
            self.element_data.append(item.data)
            return item.uid

    def get_edge(self, uid):
        if not isinstance(uid, uuid.UUID):
            raise TypeError("{} is not a uuid".format(uid))
        index = self.element2index[uid]
        edge = self.data_set.get_cell(index)
        return Edge(
            uid=uid,
            points=[self.index2point[i] for i in edge.point_ids],
            data=self.element_data[index])

    def update_edge(self, edge):
        try:
            index = self.element2index[edge.uid]
        except KeyError:
            message = "Edge with {} does exist"
            raise ValueError(message.format(edge.uid))
        point_ids = [self.point2index[uid] for uid in edge.points]
        self.elements[index] = point_ids
        self.element_data[index] = edge.data

    def has_edges(self):
        cell_types = self.data_set.cell_types_array
        for type_ in VTKEDGETYPES:
            if type_ in cell_types:
                return True
        else:
            return False

    def iter_edges(self, uids=None):
        if uids is None:
            cell_types = self.data_set.cell_types_array
            for index, cell_type in enumerate(cell_types):
                if cell_type in VTKEDGETYPES:
                    edge = self.data_set.get_cell(index)
                    yield Edge(
                        uid=self.index2element[index],
                        points=[self.index2point[i] for i in edge.point_ids],
                        data=self.element_data[index])
        else:
            for uid in uids:
                yield self.get_edge(uid)

    # Face operations ########################################################

    def get_face(self, uid):
        if not isinstance(uid, uuid.UUID):
            raise TypeError("{} is not a uuid".format(uid))
        index = self.element2index[uid]
        face = self.data_set.get_cell(index)
        return Face(
            uid=uid,
            points=[self.index2point[i] for i in face.point_ids],
            data=self.element_data[index])

    def add_face(self, face):
        data_set = self.data_set
        element2index = self.element2index
        with self._add_item(face, element2index) as item:
            point_ids = [self.point2index[uid] for uid in item.points]
            index = data_set.insert_next_cell(
                FACE2VTKCELL[len(point_ids)], point_ids)
            element2index[item.uid] = index
            self.index2element[index] = item.uid
            self.element_data.append(item.data)
            return item.uid

    def has_faces(self):
        cell_types = self.data_set.cell_types_array
        for type_ in VTKFACETYPES:
            if type_ in cell_types:
                return True
        else:
            return False

    def update_face(self, face):
        try:
            index = self.element2index[face.uid]
        except KeyError:
            message = "Face with {} does exist"
            raise ValueError(message.format(face.uid))
        point_ids = [self.point2index[uid] for uid in face.points]
        self.elements[index] = point_ids
        self.element_data[index] = face.data

    def iter_faces(self, uids=None):
        if uids is None:
            cell_types = self.data_set.cell_types_array
            for index, cell_type in enumerate(cell_types):
                if cell_type in VTKFACETYPES:
                    face = self.data_set.get_cell(index)
                    yield Face(
                        uid=self.index2element[index],
                        points=[self.index2point[i] for i in face.point_ids],
                        data=self.element_data[index])
        else:
            for uid in uids:
                yield self.get_face(uid)

    # Cell operations ########################################################

    def get_cell(self, uid):
        if not isinstance(uid, uuid.UUID):
            raise TypeError("{} is not a uuid".format(uid))
        index = self.element2index[uid]
        face = self.data_set.get_cell(index)
        return Cell(
            uid=uid,
            points=[self.index2point[i] for i in face.point_ids],
            data=self.element_data[index])

    def add_cell(self, cell):
        data_set = self.data_set
        element2index = self.element2index
        with self._add_item(cell, element2index) as item:
            point_ids = [self.point2index[uid] for uid in item.points]
            index = data_set.insert_next_cell(
                CELL2VTKCELL[len(point_ids)], point_ids)
            element2index[item.uid] = index
            self.index2element[index] = item.uid
            self.element_data.append(item.data)
            return item.uid

    def has_cells(self):
        cell_types = self.data_set.cell_types_array
        for type_ in VTKCELLTYPES:
            if type_ in cell_types:
                return True
        else:
            return False

    def update_cell(self, face):
        try:
            index = self.element2index[face.uid]
        except KeyError:
            message = "Cell with {} does exist"
            raise ValueError(message.format(face.uid))
        point_ids = [self.point2index[uid] for uid in face.points]
        self.elements[index] = point_ids
        self.element_data[index] = face.data

    def iter_cells(self, uids=None):
        if uids is None:
            cell_types = self.data_set.cell_types_array
            for index, cell_type in enumerate(cell_types):
                if cell_type in VTKCELLTYPES:
                    cell = self.data_set.get_cell(index)
                    yield Cell(
                        uid=self.index2element[index],
                        points=[self.index2point[i] for i in cell.point_ids],
                        data=self.element_data[index])
        else:
            for uid in uids:
                yield self.get_cell(uid)

    # Private interface ######################################################

    @contextlib.contextmanager
    def _add_item(self, item, container):
        if item.uid is None:
            item.uid = uuid.uuid4()
        elif item.uid in container:
            message = "Item with id:{} already exists"
            raise ValueError(message.format(item.uid))
        yield item
<commit_msg>consolidate duplicate functionality<commit_after>import uuid
import contextlib
import copy

from tvtk.api import tvtk

from simphony.cuds.abstractmesh import ABCMesh
from simphony.cuds.mesh import Point, Edge, Face, Cell
from simphony.core.data_container import DataContainer
from simphony_mayavi.core.api import (
    CubaData, CellCollection, supported_cuba, mergedocs,
    EDGE2VTKCELL, FACE2VTKCELL, CELL2VTKCELL,
    ELEMENT2VTKCELLTYPES, VTKCELLTYPE2ELEMENT)

@mergedocs(ABCMesh)
class VTKMesh(ABCMesh):

    def __init__(self, name, data=None, data_set=None):
        self.name = name
        self._data = DataContainer() if data is None else DataContainer(data)
        #: The mapping from uid to point index
        self.point2index = {}
        #: The reverse mapping from index to point uid
        self.index2point = {}
        #: The mapping from uid to bond index
        self.element2index = {}
        #: The reverse mapping from index to bond uid
        self.index2element = {}

        # Setup the data_set
        if data_set is None:
            points = tvtk.Points()
            # Need to initialise lines with empty so that we
            # do not get the shared CellArray
            data_set = tvtk.UnstructuredGrid(points=points)
        else:
            for index in xrange(data_set.number_of_points):
                uid = uuid.uuid4()
                self.point2index[uid] = index
                self.index2point[index] = uid
            for index in xrange(data_set.number_of_cells):
                uid = uuid.uuid4()
                self.element2index[uid] = index
                self.index2element[index] = uid

        #: The vtk.PolyData dataset
        self.data_set = data_set

        #: The currently supported and stored CUBA keywords.
        self.supported_cuba = supported_cuba()

        #: Easy access to the vtk PointData structure
        data = data_set.point_data
        if data.number_of_arrays == 0:
            size = data_set.number_of_points
        else:
            size = None
        self.point_data = CubaData(
            data, stored_cuba=self.supported_cuba, size=size)

        #: Easy access to the vtk CellData structure
        data = data_set.cell_data
        ncells = data_set.number_of_cells
        if data.number_of_arrays == 0 and ncells != 0:
            size = ncells
        else:
            size = None
        self.element_data = CubaData(
            data, stored_cuba=self.supported_cuba, size=size)

        # Elements cells
        self.elements = CellCollection(data_set.get_cells())

    @property
    def data(self):
        """ The container data
        """
        return DataContainer(self._data)

    @data.setter
    def data(self, value):
        self._data = DataContainer(value)

    # Point operations ####################################################

    def add_point(self, point):
        data_set = self.data_set
        points = data_set.points
        point2index = self.point2index
        with self._add_item(point, point2index) as item:
            index = points.insert_next_point(item.coordinates)
            point2index[item.uid] = index
            self.index2point[index] = item.uid
            self.point_data.append(item.data)
            return item.uid

    def get_point(self, uid):
        if not isinstance(uid, uuid.UUID):
            raise TypeError("{} is not a uuid".format(uid))
        index = int(self.point2index[uid])
        return Point(
            uid=uid,
            coordinates=self.data_set.points[index],
            data=self.point_data[index])

    def update_point(self, point):
        try:
            index = self.point2index[point.uid]
        except KeyError:
            message = "Point with {} does exist"
            raise ValueError(message.format(point.uid))
        # Need to cast to int https://github.com/enthought/mayavi/issues/173
        self.data_set.points[int(index)] = point.coordinates
        self.point_data[index] = point.data

    def iter_points(self, uids=None):
        if uids is None:
            for uid in self.point2index:
                yield self.get_point(uid)
        else:
            for uid in uids:
                yield self.get_point(uid)

    # special private ########################################################

    def _update_element(self, element):
        try:
            index = self.element2index[element.uid]
        except KeyError:
            message = "{} with {} does exist"
            raise ValueError(message.format(type(element), element.uid))
        point_ids = [self.point2index[uid] for uid in element.points]
        self.elements[index] = point_ids
        self.element_data[index] = element.data

    # Edge operations ########################################################

    def get_edge(self, uid):
        if not isinstance(uid, uuid.UUID):
            raise TypeError("{} is not a uuid".format(uid))
        index = self.element2index[uid]
        return self._get_element(index, Edge)

    def has_edges(self):
        return self._has_elements(Edge)

    def iter_edges(self, uids=None):
        if uids is None:
            for edge in self._iter_elements(Edge):
                yield edge
        else:
            for uid in uids:
                yield self.get_edge(uid)

    def add_edge(self, edge):
        return self._add_element(edge, mapping=EDGE2VTKCELL)

    update_edge = copy.copy(_update_element)

    # Face operations ########################################################

    def get_face(self, uid):
        if not isinstance(uid, uuid.UUID):
            raise TypeError("{} is not a uuid".format(uid))
        index = self.element2index[uid]
        return self._get_element(index, Face)

    def has_faces(self):
        return self._has_elements(Face)

    def iter_faces(self, uids=None):
        if uids is None:
            for face in self._iter_elements(Face):
                yield face
        else:
            for uid in uids:
                yield self.get_face(uid)

    def add_face(self, face):
        return self._add_element(face, mapping=FACE2VTKCELL)

    update_face = copy.copy(_update_element)

    # Cell operations ########################################################

    def get_cell(self, uid):
        if not isinstance(uid, uuid.UUID):
            raise TypeError("{} is not a uuid".format(uid))
        index = self.element2index[uid]
        return self._get_element(index, Cell)

    def has_cells(self):
        return self._has_elements(Cell)

    def iter_cells(self, uids=None):
        if uids is None:
            for cell in self._iter_elements(Cell):
                yield cell
        else:
            for uid in uids:
                yield self.get_cell(uid)

    def add_cell(self, cell):
        return self._add_element(cell, mapping=CELL2VTKCELL)

    update_cell = copy.copy(_update_element)

    # Private interface ######################################################

    @contextlib.contextmanager
    def _add_item(self, item, container):
        if item.uid is None:
            item.uid = uuid.uuid4()
        elif item.uid in container:
            message = "Item with id:{} already exists"
            raise ValueError(message.format(item.uid))
        yield item

    def _has_elements(self, element):
        cell_types = self.data_set.cell_types_array
        for type_ in ELEMENT2VTKCELLTYPES[element]:
            if type_ in cell_types:
                return True
        else:
            return False

    def _get_element(self, index, type_=None):
        data_set = self.data_set
        element = data_set.get_cell(index)
        if type_ is None:
            type_ = VTKCELLTYPE2ELEMENT[data_set.get_cell_type(index)]
        return type_(
            uid=self.index2element[index],
            points=[self.index2point[i] for i in element.point_ids],
            data=self.element_data[index])

    def _iter_elements(self, type_):
        cell_types = self.data_set.cell_types_array
        types = ELEMENT2VTKCELLTYPES[type_]
        index2point = self.index2point
        index2element = self.index2element
        element_data = self.element_data
        data_set = self.data_set
        for index, cell_type in enumerate(cell_types):
            if cell_type in types:
                element = data_set.get_cell(index)
                yield type_(
                    uid=index2element[index],
                    points=[index2point[i] for i in element.point_ids],
                    data=element_data[index])

    def _add_element(self, element, mapping):
        data_set = self.data_set
        element2index = self.element2index
        with self._add_item(element, element2index) as item:
            point_ids = [self.point2index[uid] for uid in item.points]
            index = data_set.insert_next_cell(
                mapping[len(point_ids)], point_ids)
            element2index[item.uid] = index
            self.index2element[index] = item.uid
            self.element_data.append(item.data)
            return item.uid
<|endoftext|>


<commit_before>import utils
import blockio

import dag
import os
import re

import nbformat
from nbformat.v4 import new_markdown_cell, new_code_cell, new_notebook

class FlowManager:

    def __init__(self, dag, path):
        self._dag = dag
        self._path = path

    @staticmethod
    def read_nb(fpathname):
        with open(fpathname, encoding='utf-8') as f:
            return nbformat.read(f, as_version=4)

    def _read_block_nb(self, block_id):
        fname = self._dag.block_filename(block_id)
        fpathname = os.path.join(self._path, fname)
        return self.read_nb(fpathname)

    def _header_cell(self):
        source = open("header_cell.py").read()
        return new_code_cell(source = source, metadata = {'dagpy': {'cell_type': blockio.HEADER_CELL_TYPE}})

    def _delimiter_cell(self, block_id):
        source = '### Block_id\n' + str(block_id) + '\n'
        source += '##### Description\n' + self._dag.get_block_att(block_id, 'description', default='') + '\n'
        source += '##### Parents\n' + ', '.join(self._dag.get_block_att(block_id, 'parents', default='')) + '\n'
        source += '##### Filter\n' + ', '.join(self._dag.get_block_att(block_id, 'filter', default='')) + '\n'
        source += '##### File\n' + self._dag.get_block_att(block_id, 'file', default='')
        return new_markdown_cell(source=source, metadata={'dagpy': {'cell_type': blockio.DELIMITER_CELL_TYPE, 'block_id': block_id}})

    def _create_notebook(self):
        title_cell = new_markdown_cell(source='## DAGpy Flow notebook\nIf you want to start a new block, just run the start_new_dagpy_block() in the cell!', metadata={'dagpy': {'cell_type': blockio.TITLE_CELL_TYPE}})
        return new_notebook(cells=[self._header_cell(), title_cell])

    def merge_blocks(self, block_ids):
        merged = self._create_notebook()
        for block_id in block_ids:
            nb = self._read_block_nb(block_id)
            merged.cells.append(self._delimiter_cell(block_id))
            merged.cells.extend(nb.cells)
        merged.metadata.name = "dagpy_flow"
        return merged

    def flow_to_file(self, block_ids, filepathname):
        block_ids_linearized = utils.linearize_dependencies(self._dag, block_ids)
        nb = self.merge_blocks(block_ids_linearized)
        nbformat.write(nb, filepathname)

    @staticmethod
    def parse_delimiter_cell_att(att_name, lines):
        if att_name == 'block_id':
            return re.sub( '\s+', '', ''.join(lines))
        if att_name == 'description':
            return '\n'.join(lines)
        if att_name == 'file':
            return ''.join([line.strip() for line in lines])
        if att_name in ['parents', 'filter']:
            return re.sub( '[,\s]+', ' ', ' '.join(lines)).split()

    @staticmethod
    def parse_delimiter_cell(cell):
        block_meta = {}
        lines = cell.source.splitlines()
        att_name = None
        att_lines = []
        for line in lines:
            if line and line[0] == '#':
                if att_name:
                    block_meta[att_name] = FlowManager.parse_delimiter_cell_att(att_name, att_lines)

                att_name = re.sub( '[#\s]+', '', line).strip().lower()
                att_lines = []
            elif att_name:
                att_lines += [line]
        if att_name:
            block_meta[att_name] = FlowManager.parse_delimiter_cell_att(att_name, att_lines)

        if 'block_id' in cell.metadata['dagpy']:
            # preexisting block
            if 'block_id' in block_meta and block_meta['block_id'] != cell.metadata['dagpy']['block_id']:
                # user has changed the block id
                # TODO: Decide if block_id can be changed in flow. Probably not.
                block_meta['block_id'] = cell.metadata['dagpy']['block_id']
            else:
                block_meta['block_id'] = cell.metadata['dagpy']['block_id']
        else:
            # new block
            if 'block_id' not in block_meta:
                print('ERROR: no block_id specified')
                block_meta['block_id'] = 'new_block'

        return block_meta

    def apply_flow_changes(self, filepathname):
        cells = self.read_nb(filepathname).cells
        blocks_to_save = {}
        block_cells = []
        block_meta = {}
        blocks_meta = []
        for cell in cells:
            if 'dagpy' in cell.metadata:
                if cell.metadata['dagpy']['cell_type'] == blockio.DELIMITER_CELL_TYPE:  # other dagpy cell types are ignored
                    if block_meta:
                        blocks_meta += [block_meta]
                        blocks_to_save[block_meta['block_id']] = block_cells
                        block_cells = []
                    block_meta = FlowManager.parse_delimiter_cell(cell)
            else:
                block_cells += [cell]

        if block_meta:
            blocks_meta += [block_meta]
            blocks_to_save[block_meta['block_id']] = block_cells

        if blocks_meta:
            self._dag.update_blocks(blocks_meta)

        for block_id, block_cells in blocks_to_save.items():
            blockio.save_block(block_id, block_cells, self._dag)
<commit_msg>Update flowmanager.py<commit_after>import utils
import blockio

import dag
import os
import re

import nbformat
from nbformat.v4 import new_markdown_cell, new_code_cell, new_notebook

class FlowManager:
    """Manages the creation, parsing and saving of flow notebooks"""

    def __init__(self, dag, path):
        """Init from DAG object used for block lookup and .dagpy path to which block files are relative."""
        self._dag = dag
        self._path = path

    @staticmethod
    def read_nb(fpathname):
        """Read the iPython notebook file."""
        with open(fpathname, encoding='utf-8') as f:
            return nbformat.read(f, as_version=4)

    def _read_block_nb(self, block_id):
        """Read the block notebook file."""
        fname = self._dag.block_filename(block_id)
        fpathname = os.path.join(self._path, fname)
        return self.read_nb(fpathname)

    def _header_cell(self):
        """Create the header cell with custom DAGpy code, such as the start_new_dagpy_block function.
        Header cell is loaded from the header_cell.py file.
        """
        source = open("header_cell.py").read()
        return new_code_cell(source = source, metadata = {'dagpy': {'cell_type': blockio.HEADER_CELL_TYPE}})

    def _delimiter_cell(self, block_id):
        """Delimiter cell between blocks, specifying the following block metadata."""
        source = '### Block_id\n' + str(block_id) + '\n'
        source += '##### Description\n' + self._dag.get_block_att(block_id, 'description', default='') + '\n'
        source += '##### Parents\n' + ', '.join(self._dag.get_block_att(block_id, 'parents', default='')) + '\n'
        source += '##### Filter\n' + ', '.join(self._dag.get_block_att(block_id, 'filter', default='')) + '\n'
        source += '##### File\n' + self._dag.get_block_att(block_id, 'file', default='')
        return new_markdown_cell(source=source, metadata={'dagpy': {'cell_type': blockio.DELIMITER_CELL_TYPE, 'block_id': block_id}})

    def _create_notebook(self):
        """Create new flow notebook with title and header cells"""
        title_cell = new_markdown_cell(source='## DAGpy Flow notebook\nIf you want to start a new block, just run the start_new_dagpy_block() in the cell!', metadata={'dagpy': {'cell_type': blockio.TITLE_CELL_TYPE}})
        return new_notebook(cells=[self._header_cell(), title_cell])

    def merge_blocks(self, block_ids):
        """Merge blocks into a flow notebook."""
        merged = self._create_notebook()
        for block_id in block_ids:
            nb = self._read_block_nb(block_id)
            merged.cells.append(self._delimiter_cell(block_id))
            merged.cells.extend(nb.cells)
        merged.metadata.name = "dagpy_flow"
        return merged

    def flow_to_file(self, block_ids, filepathname):
        """Create flow and save it to file."""
        block_ids_linearized = utils.linearize_dependencies(self._dag, block_ids)
        nb = self.merge_blocks(block_ids_linearized)
        nbformat.write(nb, filepathname)

    @staticmethod
    def parse_delimiter_cell_att(att_name, lines):
        """Parse a single block attribute from a delimiter cell."""
        if att_name == 'block_id':
            return re.sub( '\s+', '', ''.join(lines))
        if att_name == 'description':
            return '\n'.join(lines)
        if att_name == 'file':
            return ''.join([line.strip() for line in lines])
        if att_name in ['parents', 'filter']:
            return re.sub( '[,\s]+', ' ', ' '.join(lines)).split()

    @staticmethod
    def parse_delimiter_cell(cell):
        """Parse a delimiter cell to retrieve block metadata attributes."""
        block_meta = {}
        lines = cell.source.splitlines()
        att_name = None
        att_lines = []
        for line in lines:
            if line and line[0] == '#':
                if att_name:
                    block_meta[att_name] = FlowManager.parse_delimiter_cell_att(att_name, att_lines)

                att_name = re.sub( '[#\s]+', '', line).strip().lower()
                att_lines = []
            elif att_name:
                att_lines += [line]
        if att_name:
            block_meta[att_name] = FlowManager.parse_delimiter_cell_att(att_name, att_lines)

        if 'block_id' in cell.metadata['dagpy']:
            # preexisting block
            if 'block_id' in block_meta and block_meta['block_id'] != cell.metadata['dagpy']['block_id']:
                # user has changed the block id
                # TODO: Decide if block_id can be changed in flow. Probably not.
                block_meta['block_id'] = cell.metadata['dagpy']['block_id']
            else:
                block_meta['block_id'] = cell.metadata['dagpy']['block_id']
        else:
            # new block
            if 'block_id' not in block_meta:
                print('ERROR: no block_id specified')
                block_meta['block_id'] = 'new_block'

        return block_meta

    def apply_flow_changes(self, filepathname):
        """Apply changes made in the flow to block files and the DAG."""
        cells = self.read_nb(filepathname).cells
        blocks_to_save = {}
        block_cells = []
        block_meta = {}
        blocks_meta = []
        for cell in cells:
            if 'dagpy' in cell.metadata:
                if cell.metadata['dagpy']['cell_type'] == blockio.DELIMITER_CELL_TYPE:  # other dagpy cell types are ignored
                    if block_meta:
                        blocks_meta += [block_meta]
                        blocks_to_save[block_meta['block_id']] = block_cells
                        block_cells = []
                    block_meta = FlowManager.parse_delimiter_cell(cell)
            else:
                block_cells += [cell]

        if block_meta:
            blocks_meta += [block_meta]
            blocks_to_save[block_meta['block_id']] = block_cells

        if blocks_meta:
            self._dag.update_blocks(blocks_meta)

        for block_id, block_cells in blocks_to_save.items():
            blockio.save_block(block_id, block_cells, self._dag)
<|endoftext|>


<commit_before>from threading import RLock

def Synchronized(arg):
    def decorator(fn):
        lock = RLock()
        def wrapped(*args, **kwargs):
            lock.acquire()
            try:
                return fn(*args, **kwargs)
            finally:
                lock.release()

        return wrapped

    return decorator
<commit_msg>[-] synchronized fixed<commit_after>from threading import RLock

def Synchronized(lock=None):
    if not lock:
        lock=RLock()
    def decorator(fn):
        def wrapped(*args, **kwargs):
            lock.acquire()
            try:
                return fn(*args, **kwargs)
            finally:
                lock.release()

        return wrapped

    return decorator
<|endoftext|>


<commit_before># coding=utf-8

from typing import cast

from twilio.base import TwilioRestException
from twilio.rest import Client

from ..Monitors.monitor import Monitor
from .alerter import Alerter, AlertLength, AlertType, register

@register
class TwilioSMSAlerter(Alerter):
    """Send SMS alerts using Twilio."""

    alerter_type = "twilio_sms"

    def __init__(self, config_options: dict) -> None:
        super().__init__(config_options)
        account_sid = cast(
            str, self.get_config_option("account_sid", required=True, allow_empty=False)
        )
        auth_token = cast(
            str, self.get_config_option("auth_token", required=True, allow_empty=False)
        )
        self.target = cast(
            str, self.get_config_option("target", required=True, allow_empty=False)
        )
        self.client = Client(account_sid, auth_token)

        self.sender = cast(str, self.get_config_option("sender", default="SmplMntr"))
        if len(self.sender) > 11:
            self.alerter_logger.warning("truncating SMS sender name to 11 chars")
            self.sender = self.sender[:11]

        self.support_catchup = True

    def send_alert(self, name: str, monitor: Monitor) -> None:
        """Send an SMS alert."""

        if not monitor.urgent:
            return

        alert_type = self.should_alert(monitor)
        if alert_type not in [AlertType.FAILURE, AlertType.SUCCESS]:
            return

        message = self.build_message(AlertLength.SMS, alert_type, monitor)

        params = {
            "body": message,
            "to": self.target,
            "from_": self.sender,
        }

        if not self._dry_run:
            try:
                self.client.messages.create(**params)
            except TwilioRestException:
                self.alerter_logger.exception("SMS sending failed")
        else:
            self.alerter_logger.info(
                "dry_run: would send SMS with Twilio: %s", str(params)
            )

    def _describe_action(self) -> str:
        return "SMSing {target} via Twilio".format(target=self.target)
<commit_msg>Fix twilio exception import<commit_after># coding=utf-8

from typing import cast

from twilio.base.exceptions import TwilioRestException
from twilio.rest import Client

from ..Monitors.monitor import Monitor
from .alerter import Alerter, AlertLength, AlertType, register

@register
class TwilioSMSAlerter(Alerter):
    """Send SMS alerts using Twilio."""

    alerter_type = "twilio_sms"

    def __init__(self, config_options: dict) -> None:
        super().__init__(config_options)
        account_sid = cast(
            str, self.get_config_option("account_sid", required=True, allow_empty=False)
        )
        auth_token = cast(
            str, self.get_config_option("auth_token", required=True, allow_empty=False)
        )
        self.target = cast(
            str, self.get_config_option("target", required=True, allow_empty=False)
        )
        self.client = Client(account_sid, auth_token)

        self.sender = cast(str, self.get_config_option("sender", default="SmplMntr"))
        if len(self.sender) > 11:
            self.alerter_logger.warning("truncating SMS sender name to 11 chars")
            self.sender = self.sender[:11]

        self.support_catchup = True

    def send_alert(self, name: str, monitor: Monitor) -> None:
        """Send an SMS alert."""

        if not monitor.urgent:
            return

        alert_type = self.should_alert(monitor)
        if alert_type not in [AlertType.FAILURE, AlertType.SUCCESS]:
            return

        message = self.build_message(AlertLength.SMS, alert_type, monitor)

        params = {
            "body": message,
            "to": self.target,
            "from_": self.sender,
        }

        if not self._dry_run:
            try:
                self.client.messages.create(**params)
            except TwilioRestException:
                self.alerter_logger.exception("SMS sending failed")
        else:
            self.alerter_logger.info(
                "dry_run: would send SMS with Twilio: %s", str(params)
            )

    def _describe_action(self) -> str:
        return "SMSing {target} via Twilio".format(target=self.target)
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- encoding: utf-8 -*-

"""
Dumps texts from typeracerdata.com.
"""

import hashlib
import os
import json
import urllib2
from HTMLParser import HTMLParser

def normalize(s):
    # Unescape html
    s = s.decode("utf-8")
    s = HTMLParser().unescape(s)
    s = s.replace("\n", " ")
    s = s.replace("\r", " ")

    while "  " in s:
        s = s.replace("  ", " ")

    return s.strip()

def loadurl(url):
    cache = os.path.join("cache", hashlib.sha1(url).hexdigest())
    if os.path.isfile(cache):
        with open(cache, "rt") as f:
            return f.read()

    html = urllib2.urlopen(url).read()

    with open(cache, "wt") as f:
        f.write(html)

    return html

def get_texts():
    return loadurl("http://www.typeracerdata.com/texts")

def get_text(text_id):
    url = "http://typeracerdata.com/text?id=%s" % text_id
    html = loadurl(url)
    return html

def process(text_id, html):
    """

Text #3550073

 Plato is always concerned to advocate views that will make people what he
 thinks virtuous; he is hardly ever intellectually honest, because he allows
 himself to judge doctrines by their social consequences.

- from The History of Western Philosophy, a book Bertrand
Russel

    """
    def skip(s, tag):
        pos = s.index(tag)
        assert(pos >= 0)
        return s[pos + len(tag):]

    def extract(s, tag):
        pos = s.index(tag)
        assert(pos >= 0)
        return s[:pos]

    html = skip(skip(html, "
"), "
")
    quote = extract(html, "
").strip()

    html = skip(html, "")
    title = extract(html, "").strip()
    html = skip(html, "")

    author = extract(html, "
").strip()
    while author[0] in ", ":
        author = author[1:]
    author = author.strip()

    prefixes = (
        "a book",
        "a movie",
        "a song",
        "by",
    )
    for p in prefixes:
        if author.startswith(p):
            author = author[len(p):].strip()

    author = normalize(author)
    title = normalize(title)
    quote = normalize(quote)

    return author, title, quote

def main():
    print("Reading main list of quotes")
    html = loadurl("http://www.typeracerdata.com/texts")

    ids = set()
    for line in html.split("\n"):
        find = "/text?id="
        if not find in line:
            continue
        start = line.index(find)
        stop = line.index('"', start + len(find))
        text_id = line[start + len(find):stop]
        ids.add(text_id)

    print("Found %d quotes" % len(ids))

    # TODO: Do in parallel
    quotes = []
    try:
        for text_id in ids:
            html = get_text(text_id)
            try:
                author, title, quote = process(text_id, html)
                quotes.append({
                    "author": author,
                    "title": title,
                    "text": quote,
                })
                print("** id %s" % text_id)
                print("\"%s\" by %s" % (title, author))
                print("\"%s\"" % quote)
                print("")
            except Exception as e:
                print(e)
                print("----")
                print(text_id)
                print(html)
                raise
    except KeyboardInterrupt:
        pass
    print("Saving quotes.json")
    with open("quotes.json", "wt") as f:
        json.dump(quotes, f)

if __name__ == "__main__":
    main()
<commit_msg>Adds merging code<commit_after>#! /usr/bin/env python
# -*- encoding: utf-8 -*-

"""
Dumps texts from typeracerdata.com.
"""

import hashlib
import os
import json
import urllib2
from HTMLParser import HTMLParser

def normalize(s):
    # Unescape html
    s = s.decode("utf-8")
    s = HTMLParser().unescape(s)
    s = s.replace("\n", " ")
    s = s.replace("\r", " ")

    while "  " in s:
        s = s.replace("  ", " ")

    return s.strip()

def loadurl(url):
    cache = os.path.join("cache", hashlib.sha1(url).hexdigest())
    if os.path.isfile(cache):
        with open(cache, "rt") as f:
            return f.read()

    html = urllib2.urlopen(url).read()

    with open(cache, "wt") as f:
        f.write(html)

    return html

def get_texts():
    return loadurl("http://www.typeracerdata.com/texts")

def get_text(text_id):
    url = "http://typeracerdata.com/text?id=%s" % text_id
    html = loadurl(url)
    return html

def process(text_id, html):
    """

Text #3550073

 Plato is always concerned to advocate views that will make people what he
 thinks virtuous; he is hardly ever intellectually honest, because he allows
 himself to judge doctrines by their social consequences.

- from The History of Western Philosophy, a book Bertrand
Russel

    """
    def skip(s, tag):
        pos = s.index(tag)
        assert(pos >= 0)
        return s[pos + len(tag):]

    def extract(s, tag):
        pos = s.index(tag)
        assert(pos >= 0)
        return s[:pos]

    html = skip(skip(html, "
"), "
")
    quote = extract(html, "
").strip()

    html = skip(html, "")
    title = extract(html, "").strip()
    html = skip(html, "")

    author = extract(html, "
").strip()
    while author[0] in ", ":
        author = author[1:]
    author = author.strip()

    prefixes = (
        "a book",
        "a movie",
        "a song",
        "a television series",
        "by",
    )
    for p in prefixes:
        if author.lower().startswith(p.lower()):
            author = author[len(p):].strip()

    author = normalize(author)
    title = normalize(title)
    quote = normalize(quote)

    return author, title, quote

def main():
    print("Reading main list of quotes")
    html = loadurl("http://www.typeracerdata.com/texts")

    ids = set()
    for line in html.split("\n"):
        find = "/text?id="
        if not find in line:
            continue
        start = line.index(find)
        stop = line.index('"', start + len(find))
        text_id = line[start + len(find):stop]
        ids.add(text_id)

    print("Found %d quotes" % len(ids))

    # TODO: Do in parallel
    quotes = []
    try:
        for text_id in ids:
            html = get_text(text_id)
            try:
                author, title, quote = process(text_id, html)
                quotes.append({
                    "author": author,
                    "title": title,
                    "text": quote,
                })
                print("** id %s" % text_id)
                print("\"%s\" by %s" % (title, author))
                print("\"%s\"" % quote)
                print("")
            except Exception as e:
                print(e)
                print("----")
                print(text_id)
                print(html)
                raise
    except KeyboardInterrupt:
        pass
    print("Saving quotes.json")

    with open("../wpm/data/examples.json", "rt") as f:
        examples = json.load(f)

    oldcount = len(examples)

    # Make quotes unique
    unique = set()
    for quote in examples + quotes:
        author = quote["author"]
        title = quote["title"]
        text = quote["text"]
        unique.add((author, title, text))

    examples = []
    for author, title, text in unique:
        examples.append({
            "author": author,
            "title": title,
            "text": text,
        })

    with open("../wpm/data/examples.json", "wt") as f:
        json.dump(examples, f)

    print("Added %d quotes for a total of %d" % (oldcount - len(quotes),
        len(examples)))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
import getopt
import string

trace_file = ""
sym_filters = []
file_filters = []

# To understand the basic working flow, let's see an example, if there is a call
# graph like:
#a
#|--b
#|  |--c
#|--b
#|  |--d
#|  |--d
#|  |--e

# And, the data sturcture will be like this:
#{
#    prefix = "xx",
#    func = "xx",
#    func_location = "xx",
#    caller = "xx",
#    caller_location = "xx",
#    times = n,
#    next = [],
#}
#
# So the full memory data will be:
#[
#    {
#        prefix = "",
#        func = "a",
#        func_location = "xx",
#        caller = "xx",
#        caller_location = "xx",
#        times = 1,
#        next = [
#            {
#                prefix = "..",
#                func = "b",
#                func_location = "xx",
#                caller = "xx",
#                caller_location = "xx",
#                times = 1,
#                next = [
#                    {
#                        prefix = "....",
#                        func = "c",
#                        func_location = "xx",
#                        caller = "xx",
#                        caller_location = "xx",
#                        times = 1,
#                        next = []
#                    }
#                ]
#            },
#            {
#                prefix = "..",
#                func = "b",
#                func_location = "xx",
#                caller = "xx",
#                caller_location = "xx",
#                times = 1,
#                next = [
#                    {
#                        prefix = "....",
#                        func = "d",
#                        func_location = "xx",
#                        caller = "xx",
#                        caller_location = "xx",
#                        times = 2,
#                        next = []
#                    },
#                    {
#                        prefix = "....",
#                        func = "e",
#                        func_location = "xx",
#                        caller = "xx",
#                        caller_location = "xx",
#                        times = 1,
#                        next = []
#                    }
#                ]
#            }
#        ]
#    }
#]

def load_trace_file():
    try:
        file = open(trace_file, "r")

        call_graph = []
        gen_report(file, "", call_graph)
        dump_graph(call_graph)
    finally:
        file.close()

def create_frame(prefix, func_name, func_location, caller_name, caller_location):
    return {
        'prefix' : prefix,
        'func_name' : func_name,
        'func_location' : func_location,
        'caller_name' : caller_name,
        'caller_location' : caller_location,
        'times' : 1,
        'next' : []
    }

def gen_report(file, prefix, call_list):
    while True:
        line = file.readline()
        if not line:
            return

        value_list = line.strip('\n').split('|')
        # get the data from one line
        type = value_list[0]
        # func_name and func_location is the primary key to identify a unique
        # function
        func_name = value_list[1]
        func_location = value_list[2]
        caller_name = value_list[3]
        caller_location = os.path.basename(value_list[4])

        # we entry into next call
        if type == "E":
            frame = create_frame(prefix, func_name, func_location, caller_name, caller_location)

            gen_report(file, frame['prefix'] + "..", frame['next'])

            # if the last frame is equal to the next_call_list, we only need to
            # increase the times counter in the frame
            if not call_list:
                call_list.append(frame)

            # compare frame with last frame exclude the times field
            # To compare it, we should set the last_frame.times = 1 first,
            # then do the comparison
            else:
                last_frame = call_list[-1]
                prev_times = last_frame['times']
                last_frame['times'] = 1

                if last_frame == frame:
                    last_frame['times'] = prev_times + 1
                else:
                    last_frame['times'] = prev_times
                    call_list.append(frame)

        # we exit from a call
        elif type == "X":
            return

# if the frame funcname is in skipping list, then return True, or False
def sym_should_skip(frame):
    raw_func_name = frame['func_name']
    func_name = ""

    # if :: in func name, that means it's C++ function
    if "::" in raw_func_name:
        func_name = raw_func_name.split("::")[0]
        func_name = func_name.split(" ")
        if len(func_name) == 2:
            func_name = func_name[1]
        else:
            func_name = func_name[0]
    else:
        func_name = raw_func_name

    if func_name in sym_filters:
        return True
    else:
        return False

def file_should_skip(frame):
    raw_func_location = frame['func_location']

    for filter_item in file_filters:
        if filter_item in raw_func_location:
            return True
        else:
            return False

def dump_graph(call_graph):
    for frame in call_graph:
        if sym_should_skip(frame):
            continue
        elif file_should_skip(frame):
            continue

        print "%s %dx %s(%s) - (called from %s)" % (frame['prefix'],
                                                    frame['times'],
                                                    frame['func_name'],
                                                    frame['func_location'],
                                                    frame['caller_location'])
        dump_graph(frame['next'])

def usage():
    print "usage: formatter.py [-f trace.txt] [-s filter[, filters...]]"

if __name__ == "__main__":
    if len(sys.argv) == 1:
        usage()
        sys.exit(1)

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hf:s:S:")

        for op, value in opts:
            if op == "-h":
                sys.exit(0)
            elif op == "-f":
                trace_file = value
            elif op == '-s':
                sym_filters = value.split(",")
            elif op == '-S':
                file_filters = value.split(",")
    except:
        usage()
        sys.exit(1)

    load_trace_file()
<commit_msg>support skip useless X lines at the top of file<commit_after>#!/usr/bin/python

import sys
import os
import getopt
import string

trace_file = ""
sym_filters = []
file_filters = []

process_start = False

# To understand the basic working flow, let's see an example, if there is a call
# graph like:
#a
#|--b
#|  |--c
#|--b
#|  |--d
#|  |--d
#|  |--e

# And, the data sturcture will be like this:
#{
#    prefix = "xx",
#    func = "xx",
#    func_location = "xx",
#    caller = "xx",
#    caller_location = "xx",
#    times = n,
#    next = [],
#}
#
# So the full memory data will be:
#[
#    {
#        prefix = "",
#        func = "a",
#        func_location = "xx",
#        caller = "xx",
#        caller_location = "xx",
#        times = 1,
#        next = [
#            {
#                prefix = "..",
#                func = "b",
#                func_location = "xx",
#                caller = "xx",
#                caller_location = "xx",
#                times = 1,
#                next = [
#                    {
#                        prefix = "....",
#                        func = "c",
#                        func_location = "xx",
#                        caller = "xx",
#                        caller_location = "xx",
#                        times = 1,
#                        next = []
#                    }
#                ]
#            },
#            {
#                prefix = "..",
#                func = "b",
#                func_location = "xx",
#                caller = "xx",
#                caller_location = "xx",
#                times = 1,
#                next = [
#                    {
#                        prefix = "....",
#                        func = "d",
#                        func_location = "xx",
#                        caller = "xx",
#                        caller_location = "xx",
#                        times = 2,
#                        next = []
#                    },
#                    {
#                        prefix = "....",
#                        func = "e",
#                        func_location = "xx",
#                        caller = "xx",
#                        caller_location = "xx",
#                        times = 1,
#                        next = []
#                    }
#                ]
#            }
#        ]
#    }
#]

def load_trace_file():
    try:
        file = open(trace_file, "r")

        call_graph = []
        gen_report(file, "", call_graph)
        dump_graph(call_graph)
    finally:
        file.close()

def create_frame(prefix, func_name, func_location, caller_name, caller_location):
    return {
        'prefix' : prefix,
        'func_name' : func_name,
        'func_location' : func_location,
        'caller_name' : caller_name,
        'caller_location' : caller_location,
        'times' : 1,
        'next' : []
    }

def should_skip_line(type):
    global process_start
    if process_start == False:
        if type == "X":
            return True
        else:
            process_start = True
            return False
    else:
        return False

def gen_report(file, prefix, call_list):
    while True:
        line = file.readline()
        if not line:
            return

        value_list = line.strip('\n').split('|')

        # get the data from one line
        type = value_list[0]

        # We should pre-cut off the top of 'X' lines, or the data will be useless
        if should_skip_line(type):
            print "skip the useless X line"
            continue

        # func_name and func_location is the primary key to identify a unique
        # function
        func_name = value_list[1]
        func_location = value_list[2]
        caller_name = value_list[3]
        caller_location = os.path.basename(value_list[4])

        # we entry into next call
        if type == "E":
            frame = create_frame(prefix, func_name, func_location, caller_name, caller_location)

            gen_report(file, frame['prefix'] + "..", frame['next'])

            # if the last frame is equal to the next_call_list, we only need to
            # increase the times counter in the frame
            if not call_list:
                call_list.append(frame)

            # compare frame with last frame exclude the times field
            # To compare it, we should set the last_frame.times = 1 first,
            # then do the comparison
            else:
                last_frame = call_list[-1]
                prev_times = last_frame['times']
                last_frame['times'] = 1

                if last_frame == frame:
                    last_frame['times'] = prev_times + 1
                else:
                    last_frame['times'] = prev_times
                    call_list.append(frame)

        # we exit from a call
        elif type == "X":
            return

# if the frame funcname is in skipping list, then return True, or False
def sym_should_skip(frame):
    raw_func_name = frame['func_name']
    func_name = ""

    # if :: in func name, that means it's C++ function
    if "::" in raw_func_name:
        func_name = raw_func_name.split("::")[0]
        func_name = func_name.split(" ")
        if len(func_name) == 2:
            func_name = func_name[1]
        else:
            func_name = func_name[0]
    else:
        func_name = raw_func_name

    if func_name in sym_filters:
        return True
    else:
        return False

def file_should_skip(frame):
    raw_func_location = frame['func_location']

    for filter_item in file_filters:
        if filter_item in raw_func_location:
            return True
        else:
            return False

def dump_graph(call_graph):
    for frame in call_graph:
        if sym_should_skip(frame):
            continue
        elif file_should_skip(frame):
            continue

        print "%s %dx %s(%s) - (called from %s)" % (frame['prefix'],
                                                    frame['times'],
                                                    frame['func_name'],
                                                    frame['func_location'],
                                                    frame['caller_location'])
        dump_graph(frame['next'])

def usage():
    print "usage: formatter.py [-f trace.txt] [-s sym_filter[, sym_filters...]] [-S file_filter[, file_filters...]]"

if __name__ == "__main__":
    if len(sys.argv) == 1:
        usage()
        sys.exit(1)

    try:
        opts, args = getopt.getopt(sys.argv[1:], "hf:s:S:")

        for op, value in opts:
            if op == "-h":
                sys.exit(0)
            elif op == "-f":
                trace_file = value
            elif op == '-s':
                sym_filters = value.split(",")
            elif op == '-S':
                file_filters = value.split(",")
    except:
        usage()
        sys.exit(1)

    load_trace_file()
<|endoftext|>


<commit_before>#!/usr/bin/python2.4
import telepathy.errors
import inspect
import gengobject

out = open("telepathy-errors.h", 'w')

gengobject.print_license(out, "telepathy-errors.h", "Header for Telepathy error types")

gengobject.print_header_begin(out, "telepathy_errors")
out.write("typedef enum\n{\n")

errors = [];
max_name_length = 0
for (cname,val) in telepathy.errors.__dict__.items():
    if inspect.isclass(val):
        if '_dbus_error_name' in val.__dict__:
            errors.append(val)
            if len(val.__name__) > max_name_length:
                max_name_length = len(val.__name__)

for val in errors:
    line = "  "+val.__name__ + ","
    line = line.ljust(max_name_length + 4)
    line += "/** "+val.__doc__.strip()
    while len (line) > 79:
        cut = line[:80]
        cutidx = cut.rfind(' ')
        if cutidx == 80:
            out.write(cut[:79])
        else:
            out.write(cut[:cutidx])
        out.write('\n')
        line = ' ' * (max_name_length +4) + ' * ' + line[cutidx:]
    out.write(line + '\n')
    out.write(' ' * (max_name_length + 4) + ' */\n')

out.write("} TelepathyErrors; \n\n")

gengobject.print_header_end(out, "telepathy_errors");
<commit_msg>added to generrors.py to generate the error quark needed to register the errors with dbus<commit_after>#!/usr/bin/python2.4
import telepathy.errors
import inspect
import gengobject

out = open("telepathy-errors.h", 'w')

gengobject.print_license(out, "telepathy-errors.h", "Header for Telepathy error types")

gengobject.print_header_begin(out, "telepathy_errors")
out.write("typedef enum\n{\n")

errors = [];
max_name_length = 0
for (cname,val) in telepathy.errors.__dict__.items():
    if inspect.isclass(val):
        if '_dbus_error_name' in val.__dict__:
            errors.append(val)
            if len(val.__name__) > max_name_length:
                max_name_length = len(val.__name__)

for val in errors:
    line = "  "+val.__name__ + ","
    line = line.ljust(max_name_length + 4)
    line += "/** "+val.__doc__.strip()
    while len (line) > 79:
        cut = line[:80]
        cutidx = cut.rfind(' ')
        if cutidx == 80:
            out.write(cut[:79])
        else:
            out.write(cut[:cutidx])
        out.write('\n')
        line = ' ' * (max_name_length +4) + ' * ' + line[cutidx:]
    out.write(line + '\n')
    out.write(' ' * (max_name_length + 4) + ' */\n')

out.write("} TelepathyErrors; \n\n")

out.write(
"""
  GQuark telepathy_errors_quark (void);
  #define TELEPATHY_ERRORS telepathy_errors_quark ()
""")

gengobject.print_header_end(out, "telepathy_errors");

out = open("telepathy-errors.c", 'w')

out.write(
"""
#include <glib.h>
#include "telepathy-errors.h"

GQuark
telepathy_errors_quark (void)
{
  static GQuark quark = 0;
  if (!quark)
    quark = g_quark_from_static_string ("telepathy_errors");
  return quark;
}
""")
<|endoftext|>


<commit_before>from os.path import abspath, join, dirname
import sys
import logging
import re
import time
import datetime

from tenyks.client import Client, run_client
from tenyks.client.config import settings

logger = logging.getLogger('tenyks-contrib.gentooservice')

class GentooService(Client):

    irc_message_filters = {
        'get_last_mention': [r'^last gentoo mention$'],
        'find_gentoo': [re.compile(r'\b(gentoo)\b', flags=re.IGNORECASE).search],
    }

    direct_only = False

    def __init__(self, *args, **kwargs):
        super(GentooService, self).__init__(*args, **kwargs)
        self.filename = 'gentoo_last_mention'

    def handle_find_gentoo(self, data, match):
        last_mention = time.time()
        with open(join(settings.DATA_WORKING_DIR, self.filename), 'w') as f:
            f.write(str(last_mention))

    def handle_get_last_mention(self, data, match):
        try:
            with open(join(settings.DATA_WORKING_DIR, self.filename), 'r') as f:
                self.send(str(datetime.datetime.fromtimestamp(float(f.read()))), data)
        except IOError:
            self.send('Wow... Never.', data)

def main():
    run_client(GentooService)

if __name__ == '__main__':
    main()
<commit_msg>funroll<commit_after>from os.path import abspath, join, dirname
import sys
import logging
import re
import time
import datetime

from tenyks.client import Client, run_client
from tenyks.client.config import settings

logger = logging.getLogger('tenyks-contrib.gentooservice')

class GentooService(Client):

    irc_message_filters = {
        'get_last_mention': [r'^last gentoo mention$'],
        'find_gentoo': [re.compile(r'\b(gentoo)\b', flags=re.IGNORECASE).search],
        'funroll': [re.compile(r'\b(funroll)\b', flags=re.IGNORECASE).search],
    }

    direct_only = False

    def __init__(self, *args, **kwargs):
        super(GentooService, self).__init__(*args, **kwargs)
        self.filename = 'gentoo_last_mention'

    def handle_find_gentoo(self, data, match):
        last_mention = time.time()
        with open(join(settings.DATA_WORKING_DIR, self.filename), 'w') as f:
            f.write(str(last_mention))

    def handle_get_last_mention(self, data, match):
        try:
            with open(join(settings.DATA_WORKING_DIR, self.filename), 'r') as f:
                self.send(str(datetime.datetime.fromtimestamp(float(f.read()))), data)
        except IOError:
            self.send('Wow... Never.', data)

    def handle_funroll(self, data, match):
        self.send('What kinda fun y\'all having?', data)

def main():
    run_client(GentooService)

if __name__ == '__main__':
    main()
<|endoftext|>