<commit_before>import os
from os import environ as env
from django.conf import global_settings
import dj_database_url
import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration

SENTRY_DSN = env.get("SENTRY_DSN", None)
if SENTRY_DSN is not None:
    sentry_sdk.init(
        dsn=SENTRY_DSN,
        integrations=[DjangoIntegration()],
        traces_sample_rate = 1.0,
    )

DEBUG = env.get('DJANGO_DEBUG', 'true') == 'true'
PROJECT_ROOT = os.path.dirname(os.path.realpath(__name__))
DATA_PATH = os.path.join(PROJECT_ROOT, "data")

ADMINS = (
    ('Adi Eyal', 'adi@openup.org.za'),
)

MANAGERS = ADMINS

DATABASES = {
    # load from DATABASE_URL env var, or default to this
    'default': dj_database_url.config(default='sqlite:///mpr.db')
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = [
    'code4sa-mpr.herokuapp.com',
    'mpr.code4sa.org',
    'mpr.openup.org.za',
    'localhost',
    '127.0.0.1',
    'medicineprices.org.za'
]

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'Africa/Johannesburg'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: '/var/www/example.com/media/'
MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: 'http://example.com/media/', 'http://media.example.com/'
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' 'static/' subdirectories and in STATICFILES_DIRS.
# Example: '/var/www/example.com/static/'
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static')

# URL prefix for static files.
# Example: 'http://example.com/static/', 'http://static.example.com/'
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like '/home/html/static' or 'C:/www/django/static'.
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(PROJECT_ROOT, 'mpr', 'static'),
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'pipeline.finders.PipelineFinder',
)

if DEBUG:
    # Make this unique, and don't share it with anybody.
    SECRET_KEY = 'ub7dk%m=d*k=aip1rl)z&v8sj&fg2msc&=km0z3u#5ct9+_43w'
else:
    SECRET_KEY = env.get('DJANGO_SECRET_KEY')

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                'mpr.context_processors.settings_context'
            ]
        }
    },
]

MIDDLEWARE = (
    'mpr.middleware.CORSMiddleware',
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'mpr.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'mpr.wsgi.application'

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'django.contrib.sitemaps',
    'pipeline',
    'mpr',
    'dataprocessing',
#    'behave_django',
)

STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'

PIPELINE = {
    'ENABLED': env.get("DJANGO_PIPELINE_ENABLED", True),
    'JAVASCRIPT': {
        'mprbase' : {
            'source_filenames': (
              'js/jquery-1.10.2.js',
              'js/bootstrap.js',
              'js/jquery.ba-hashchange.js',
              'js/medloader.js',
            ),
            'output_filename': 'js/mprbase.js',
        },
    },
    'STYLESHEETS' : {
        'mpr' : {
            'source_filenames': (
              'css/bootstrap.css',
              #'css/bootstrap-theme.css',
              'css/custom.css',
            ),
            'output_filename': 'css/mpr.css',
        }
    },
    'CSS_COMPRESSOR' : 'pipeline.compressors.yuglify.YuglifyCompressor',
    'JS_COMPRESSOR' : 'pipeline.compressors.yuglify.YuglifyCompressor',
    'DISABLE_WRAPPER' : True
}

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format' : '[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s',
            'datefmt' : '%d/%b/%Y %H:%M:%S'
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'verbose'
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'level': 'ERROR',
            'propagate': True,
        },
        'mpr': {
            'handlers': ['console'],
            'level': 'DEBUG' if DEBUG else 'ERROR',
            'propagate': True,
        },
    }
}

CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        'TIMEOUT' : 60*60*24*7
    }
}

SEGMENT_IO_KEY = env.get('SEGMENT_IO_KEY')
SESSION = 'django.contrib.sessions.backends.signed_cookies'

PRICE_PARAMETERS = {
    "VAT" : 1.15,
    "prices" : [
        (118.80, 0.46, 15.80),
        (315.53, 0.33, 30.24),
        (1104.40, 0.15, 86.11),
        (float('inf'), 0.05, 198.36),
    ]
}

LATEST_GAZETTE = "https://search.opengazettes.org.za/text/37304?dq=single%20exit%20price&page=27"

from . import loganalytics
if DEBUG:
    ANALYTICS = loganalytics.test_log_analytics
else:
    ANALYTICS = loganalytics.log_analytics
<commit_msg>Increasing maximum sentry error message length<commit_after>import os
from os import environ as env
from django.conf import global_settings
import dj_database_url
import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration
import sentry_sdk.utils

SENTRY_DSN = env.get("SENTRY_DSN", None)
if SENTRY_DSN is not None:
    sentry_sdk.init(
        dsn=SENTRY_DSN,
        integrations=[DjangoIntegration()],
        traces_sample_rate = 1.0,
    )

    sentry_sdk.utils.MAX_STRING_LENGTH = 8192

DEBUG = env.get('DJANGO_DEBUG', 'true') == 'true'
PROJECT_ROOT = os.path.dirname(os.path.realpath(__name__))
DATA_PATH = os.path.join(PROJECT_ROOT, "data")

ADMINS = (
    ('Adi Eyal', 'adi@openup.org.za'),
)

MANAGERS = ADMINS

DATABASES = {
    # load from DATABASE_URL env var, or default to this
    'default': dj_database_url.config(default='sqlite:///mpr.db')
}

# Hosts/domain names that are valid for this site; required if DEBUG is False
# See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts
ALLOWED_HOSTS = [
    'code4sa-mpr.herokuapp.com',
    'mpr.code4sa.org',
    'mpr.openup.org.za',
    'localhost',
    '127.0.0.1',
    'medicineprices.org.za'
]

# Local time zone for this installation. Choices can be found here:
# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
# although not all choices may be available on all operating systems.
# In a Windows environment this must be set to your system time zone.
TIME_ZONE = 'Africa/Johannesburg'

# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True

# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True

# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True

# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: '/var/www/example.com/media/'
MEDIA_ROOT = os.path.join(PROJECT_ROOT, 'media')

# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: 'http://example.com/media/', 'http://media.example.com/'
MEDIA_URL = '/media/'

# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' 'static/' subdirectories and in STATICFILES_DIRS.
# Example: '/var/www/example.com/static/'
STATIC_ROOT = os.path.join(PROJECT_ROOT, 'static')

# URL prefix for static files.
# Example: 'http://example.com/static/', 'http://static.example.com/'
STATIC_URL = '/static/'

# Additional locations of static files
STATICFILES_DIRS = (
    # Put strings here, like '/home/html/static' or 'C:/www/django/static'.
    # Always use forward slashes, even on Windows.
    # Don't forget to use absolute paths, not relative paths.
    os.path.join(PROJECT_ROOT, 'mpr', 'static'),
)

# List of finder classes that know how to find static files in
# various locations.
STATICFILES_FINDERS = (
    'django.contrib.staticfiles.finders.FileSystemFinder',
    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
    'pipeline.finders.PipelineFinder',
)

if DEBUG:
    # Make this unique, and don't share it with anybody.
    SECRET_KEY = 'ub7dk%m=d*k=aip1rl)z&v8sj&fg2msc&=km0z3u#5ct9+_43w'
else:
    SECRET_KEY = env.get('DJANGO_SECRET_KEY')

TEMPLATES = [
    {
        'BACKEND': 'django.template.backends.django.DjangoTemplates',
        'APP_DIRS': True,
        'OPTIONS': {
            'context_processors': [
                'django.contrib.auth.context_processors.auth',
                'django.contrib.messages.context_processors.messages',
                'mpr.context_processors.settings_context'
            ]
        }
    },
]

MIDDLEWARE = (
    'mpr.middleware.CORSMiddleware',
    'django.middleware.cache.UpdateCacheMiddleware',
    'django.middleware.common.CommonMiddleware',
    'django.contrib.sessions.middleware.SessionMiddleware',
    'django.middleware.csrf.CsrfViewMiddleware',
    'django.contrib.auth.middleware.AuthenticationMiddleware',
    'django.contrib.messages.middleware.MessageMiddleware',
    'django.middleware.cache.FetchFromCacheMiddleware',
    # Uncomment the next line for simple clickjacking protection:
    # 'django.middleware.clickjacking.XFrameOptionsMiddleware',
)

ROOT_URLCONF = 'mpr.urls'

# Python dotted path to the WSGI application used by Django's runserver.
WSGI_APPLICATION = 'mpr.wsgi.application'

INSTALLED_APPS = (
    'django.contrib.auth',
    'django.contrib.contenttypes',
    'django.contrib.sessions',
    'django.contrib.sites',
    'django.contrib.messages',
    'django.contrib.staticfiles',
    'django.contrib.admin',
    'django.contrib.sitemaps',
    'pipeline',
    'mpr',
    'dataprocessing',
#    'behave_django',
)

STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'

PIPELINE = {
    'ENABLED': env.get("DJANGO_PIPELINE_ENABLED", True),
    'JAVASCRIPT': {
        'mprbase' : {
            'source_filenames': (
              'js/jquery-1.10.2.js',
              'js/bootstrap.js',
              'js/jquery.ba-hashchange.js',
              'js/medloader.js',
            ),
            'output_filename': 'js/mprbase.js',
        },
    },
    'STYLESHEETS' : {
        'mpr' : {
            'source_filenames': (
              'css/bootstrap.css',
              #'css/bootstrap-theme.css',
              'css/custom.css',
            ),
            'output_filename': 'css/mpr.css',
        }
    },
    'CSS_COMPRESSOR' : 'pipeline.compressors.yuglify.YuglifyCompressor',
    'JS_COMPRESSOR' : 'pipeline.compressors.yuglify.YuglifyCompressor',
    'DISABLE_WRAPPER' : True
}

# A sample logging configuration. The only tangible logging
# performed by this configuration is to send an email to
# the site admins on every HTTP 500 error when DEBUG=False.
# See http://docs.djangoproject.com/en/dev/topics/logging for
# more details on how to customize your logging configuration.
LOGGING = {
    'version': 1,
    'disable_existing_loggers': False,
    'formatters': {
        'verbose': {
            'format' : '[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s',
            'datefmt' : '%d/%b/%Y %H:%M:%S'
        },
        'simple': {
            'format': '%(levelname)s %(message)s'
        },
    },
    'filters': {
        'require_debug_false': {
            '()': 'django.utils.log.RequireDebugFalse'
        }
    },
    'handlers': {
        'mail_admins': {
            'level': 'ERROR',
            'filters': ['require_debug_false'],
            'class': 'django.utils.log.AdminEmailHandler'
        },
        'console': {
            'level': 'DEBUG',
            'class': 'logging.StreamHandler',
            'formatter': 'verbose'
        },
    },
    'loggers': {
        'django': {
            'handlers': ['console'],
            'level': 'ERROR',
            'propagate': True,
        },
        'mpr': {
            'handlers': ['console'],
            'level': 'DEBUG' if DEBUG else 'ERROR',
            'propagate': True,
        },
    }
}

CACHES = {
    'default': {
        'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        'TIMEOUT' : 60*60*24*7
    }
}

SEGMENT_IO_KEY = env.get('SEGMENT_IO_KEY')
SESSION = 'django.contrib.sessions.backends.signed_cookies'

PRICE_PARAMETERS = {
    "VAT" : 1.15,
    "prices" : [
        (118.80, 0.46, 15.80),
        (315.53, 0.33, 30.24),
        (1104.40, 0.15, 86.11),
        (float('inf'), 0.05, 198.36),
    ]
}

LATEST_GAZETTE = "https://search.opengazettes.org.za/text/37304?dq=single%20exit%20price&page=27"

from . import loganalytics
if DEBUG:
    ANALYTICS = loganalytics.test_log_analytics
else:
    ANALYTICS = loganalytics.log_analytics
<|endoftext|>


<commit_before>#encoding=utf-8
import os
import pandas as pd
import numpy as np
import sys
import codecs
import glob
import click
import charset
import make_sum_data
from pyltp import SentenceSplitter

def write_list_to_file(d, file_path, verb=True):
  assert type(d) == list
  if os.path.exists(os.path.dirname(file_path)) == False:
    os.makedirs(os.path.dirname(file_path))
  f = codecs.open(file_path, "w", "utf-8")
  for x in d:
    x = str(x)
    f.write(x.strip() + "\n")
  f.close()
  if verb:
    print("write to {}".format(file_path))

@click.group()
def cli():
  pass

@click.command()
@click.argument("vocab_path")
@click.argument("save_path")
# @click.argument("--del_num/--no-del_num", default=True)
# @click.argument("--del_en/--no-del_en", default=True)
def filter_vocab(vocab_path, save_path, del_num=True, del_en=True):
  f = codecs.open(vocab_path, "r", "utf-8")
  fout = codecs.open(save_path, "w", "utf-8")
  word_sets = set()
  while True:
    line = f.readline()
    if line == "":
      break
    try:
      t = line.strip().split(" ")
      word, count = t[0], t[1]
    except Exception:
      import traceback
      traceback.print_exc()
      print(line)
      continue
    skip = False

    for index, char in enumerate(word):
        if charset.is_alphabet(char) or charset.is_number(char):
          print(word)
          skip = True
          break
        if charset.is_chinese_punctuation(char): ##solve 。榜样
          if len(word) > 1:
            print(word)
            skip = True
            break

    if skip is False:
      wst = "\t".join([word, str(count)]) + "\n"
      fout.write(wst)
  f.close()
  fout.close()

@click.command()
@click.argument("path")
@click.argument("stat_path")
@click.argument("save_path", type=str)
@click.option("--min_abs_chars", type=int, default=0, help="abs min char[0]")
@click.option("--max_abs_chars", type=int, default=int(2e5), help="max abs char[2e5]")
@click.option("--title_index", type=int, default=1, help="title index in one line[1]")
@click.option("--abs_index", type=int, default=2, help="abs index in one line[1]")
@click.option("--article_index", type=int, default=3, help="article index in one line[1]")
@click.option("--cancel_unk_label", is_flag=True, help="whether cancel null label")
@click.option("--max_cov_sents", type=int, default=None, help="abstract very high sim to article 's max sents[100]")
def filter_stat(path, stat_path, save_path, label_index=0, title_index=1, abs_index=2, article_index=3,
                min_abs_chars=0, max_abs_chars=int(2e5), abs_max_cov_sents=None, cancel_unk_label=False):
  len_stat = []
  use_samples =  0
  fin = codecs.open(path, "r", "utf-8")
  fo = codecs.open(save_path, "w", "utf-8")
  total_abs_sent_num , cov_abs_sent_num = 0 , 0
  cov_skip_abs = 0
  i = 0
  no_label_num = 0
  abs_len_filter_num = 0
  while True:
    line = fin.readline()
    if not line:
      break
    if (i+1) % int(1e4) == 0:
      print("finished {}...".format(i))
    i += 1
    t = line.strip().split("\t")
    try:
      label = None
      if label_index is not None:
        label = t[label_index]
      title = t[title_index]
      abstract = t[abs_index]
      article = t[article_index]
    except IndexError:
      continue
    abstract = make_sum_data.preprocess_abs_text(abstract)
    if len(abstract) > max_abs_chars or len(abstract) < min_abs_chars:
      abs_len_filter_num += 1
      continue
    abs_sents = SentenceSplitter.split(abstract)
    total_abs_sent_num += len(abs_sents)
    cov_num = 0
    for sent in abs_sents:
      if sent in article:
        cov_num += 1
    labeled = False
    if label is not None and "null" in label:
      labeled = True
      no_label_num +=1
    cov_abs_sent_num += cov_num
    if len(abs_sents) == cov_num or cov_num > abs_max_cov_sents:
      cov_skip_abs += 1
      continue
    if cancel_unk_label is False or labeled is True:
      use_samples += 1
      len_stat.append([len(article), len(abstract)])
      fo.write("\t".join([title, abstract, article]) + "\n")

  print("filter docs with abs len not in range[{},{})".format(abs_len_filter_num))
  print("skip {} docs with high coverage in article".format(cov_skip_abs))
  print("{} docs have no label".format(no_label_num))
  print("use:all = {}:{}".format(use_samples, i))
  print("abs-cov:all = {}:{}".format(cov_abs_sent_num, total_abs_sent_num))
  len_df = pd.DataFrame(len_stat, columns=["art_len", "abs_len"])
  print("use sample len stat")
  print(len_df.describe())
  len_df.to_csv(stat_path, index=False)
  fo.close()

cli.add_command(filter_vocab)
cli.add_command(filter_stat)

if __name__ == "__main__":
  cli()

<commit_msg>filter article<commit_after>#encoding=utf-8
import os
import pandas as pd
import numpy as np
import sys
import codecs
import glob
import click
import charset
import make_sum_data
from pyltp import SentenceSplitter

def write_list_to_file(d, file_path, verb=True):
  assert type(d) == list
  if os.path.exists(os.path.dirname(file_path)) == False:
    os.makedirs(os.path.dirname(file_path))
  f = codecs.open(file_path, "w", "utf-8")
  for x in d:
    x = str(x)
    f.write(x.strip() + "\n")
  f.close()
  if verb:
    print("write to {}".format(file_path))

@click.group()
def cli():
  pass

@click.command()
@click.argument("vocab_path")
@click.argument("save_path")
# @click.argument("--del_num/--no-del_num", default=True)
# @click.argument("--del_en/--no-del_en", default=True)
def filter_vocab(vocab_path, save_path, del_num=True, del_en=True):
  f = codecs.open(vocab_path, "r", "utf-8")
  fout = codecs.open(save_path, "w", "utf-8")
  word_sets = set()
  while True:
    line = f.readline()
    if line == "":
      break
    try:
      t = line.strip().split(" ")
      word, count = t[0], t[1]
    except Exception:
      import traceback
      traceback.print_exc()
      print(line)
      continue
    skip = False

    for index, char in enumerate(word):
        if charset.is_alphabet(char) or charset.is_number(char):
          print(word)
          skip = True
          break
        if charset.is_chinese_punctuation(char): ##solve 。榜样
          if len(word) > 1:
            print(word)
            skip = True
            break

    if skip is False:
      wst = "\t".join([word, str(count)]) + "\n"
      fout.write(wst)
  f.close()
  fout.close()

@click.command()
@click.argument("path")
@click.argument("stat_path")
@click.argument("save_path", type=str)
@click.option("--min_abs_chars", type=int, default=0, help="abs min char[0]")
@click.option("--max_abs_chars", type=int, default=int(2e5), help="max abs char[2e5]")
@click.option("--title_index", type=int, default=1, help="title index in one line[1]")
@click.option("--abs_index", type=int, default=2, help="abs index in one line[1]")
@click.option("--article_index", type=int, default=3, help="article index in one line[1]")
@click.option("--cancel_unk_label", is_flag=True, help="whether cancel null label")
@click.option("--abs_max_cov_sents", type=int, default=None, help="abstract very high sim to article 's max sents[100]")
def filter_stat(path, stat_path, save_path, label_index=0, title_index=1, abs_index=2, article_index=3,
                min_abs_chars=0, max_abs_chars=int(2e5), abs_max_cov_sents=None, cancel_unk_label=False):
  len_stat = []
  use_samples =  0
  fin = codecs.open(path, "r", "utf-8")
  fo = codecs.open(save_path, "w", "utf-8")
  total_abs_sent_num , cov_abs_sent_num = 0 , 0
  cov_skip_abs = 0
  i = 0
  no_label_num = 0
  abs_len_filter_num = 0
  while True:
    line = fin.readline()
    if not line:
      break
    if (i+1) % int(1e4) == 0:
      print("finished {}...".format(i))
    i += 1
    t = line.strip().split("\t")
    try:
      label = None
      if label_index is not None:
        label = t[label_index]
      title = t[title_index]
      abstract = t[abs_index]
      article = t[article_index]
    except IndexError:
      continue
    abstract = make_sum_data.preprocess_abs_text(abstract)
    if len(abstract) > max_abs_chars or len(abstract) < min_abs_chars:
      abs_len_filter_num += 1
      continue
    abs_sents = SentenceSplitter.split(abstract)
    total_abs_sent_num += len(abs_sents)
    cov_num = 0
    for sent in abs_sents:
      if sent in article:
        cov_num += 1
    labeled = False
    if label is not None and "null" in label:
      labeled = True
      no_label_num +=1
    cov_abs_sent_num += cov_num
    if len(abs_sents) == cov_num or cov_num > abs_max_cov_sents:
      cov_skip_abs += 1
      continue
    if cancel_unk_label is False or labeled is True:
      use_samples += 1
      len_stat.append([len(article), len(abstract)])
      fo.write("\t".join([title, abstract, article]) + "\n")

  print("filter docs with abs len not in range[{},{})".format(abs_len_filter_num))
  print("skip {} docs with high coverage in article".format(cov_skip_abs))
  print("{} docs have no label".format(no_label_num))
  print("use:all = {}:{}".format(use_samples, i))
  print("abs-cov:all = {}:{}".format(cov_abs_sent_num, total_abs_sent_num))
  len_df = pd.DataFrame(len_stat, columns=["art_len", "abs_len"])
  print("use sample len stat")
  print(len_df.describe())
  len_df.to_csv(stat_path, index=False)
  fo.close()

cli.add_command(filter_vocab)
cli.add_command(filter_stat)

if __name__ == "__main__":
  cli()

<|endoftext|>


<commit_before>from tools import pipeline_helpers
import pandas as pd

def run1(infile, features_label, output_label):
    """
    Handle variant data by only keeping rows where 10-90% of samples have
    variants.

    For CNV data, don't do any filtering.

    Otherwise, simply remove rows with zero variance.
    """
    if (features_label == 'exome_variants' or 'variants' in output_label):
        d = pipeline_helpers.remove_nfrac_variants(infile, nfrac=0.1)
    elif features_label == 'cnv':
        return pd.read_table(infile, index_col=0)
    else:
        d = pipeline_helpers.remove_zero_variance(infile)
    return d

<commit_msg>use .dropna() in filter function<commit_after>from tools import pipeline_helpers
import pandas as pd

def run1(infile, features_label, output_label):
    """
    Handle variant data by only keeping rows where 10-90% of samples have
    variants.

    For CNV data, don't do any filtering.

    Otherwise, simply remove rows with zero variance.
    """
    if (features_label == 'exome_variants' or 'variants' in output_label):
        d = pipeline_helpers.remove_nfrac_variants(infile, nfrac=0.1)
    elif features_label == 'cnv':
        return pd.read_table(infile, index_col=0)
    else:
        d = pipeline_helpers.remove_zero_variance(infile)
    return d.dropna()

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Original Perl version by: John Gruber http://daringfireball.net/ 10 May 2008
Python version by Stuart Colville http://muffinresearch.co.uk
License: http://www.opensource.org/licenses/mit-license.php
"""

import argparse
import logging
import os
import pathlib
import re
import string
import sys

import regex

__all__ = ['titlecase']
__version__ = '1.0.0'

SMALL = r'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?'
PUNCT = r"""!"“#$%&'‘()*+,\-–‒—―./:;?@[\\\]_`{|}~"""

SMALL_WORDS = regex.compile(r'^(%s)$' % SMALL, regex.I)
INLINE_PERIOD = regex.compile(r'[\p{Letter}][.][\p{Letter}]', regex.I)
UC_ELSEWHERE = regex.compile(r'[%s]*?[\p{Letter}]+[\p{Uppercase_Letter}]+?' % PUNCT)
CAPFIRST = regex.compile(r"^[%s]*?([\p{Letter}])" % PUNCT)
SMALL_FIRST = regex.compile(r'^([%s]*)(%s)\b' % (PUNCT, SMALL), regex.I)
SMALL_LAST = regex.compile(r'\b(%s)[%s]?$' % (SMALL, PUNCT), regex.I)
SUBPHRASE = regex.compile(r'([:.;?!\-–‒—―][ ])(%s)' % SMALL)
APOS_SECOND = regex.compile(r"^[dol]{1}['‘]{1}[\p{Letter}]+(?:['s]{2})?$", regex.I)
UC_INITIALS = regex.compile(r"^(?:[\p{Uppercase_Letter}]{1}\.{1}|[\p{Uppercase_Letter}]{1}\.{1}[\p{Uppercase_Letter}]{1})+$")
MAC_MC = regex.compile(r"^([Mm]c|MC)(\w.+)")

class Immutable(object):
    pass

class ImmutableString(str, Immutable):
    pass

class ImmutableBytes(bytes, Immutable):
    pass

def _mark_immutable(text):
    if isinstance(text, bytes):
        return ImmutableBytes(text)
    return ImmutableString(text)

def set_small_word_list(small=SMALL):
    global SMALL_WORDS
    global SMALL_FIRST
    global SMALL_LAST
    global SUBPHRASE
    SMALL_WORDS = regex.compile(r'^(%s)$' % small, regex.I)
    SMALL_FIRST = regex.compile(r'^([%s]*)(%s)\b' % (PUNCT, small), regex.I)
    SMALL_LAST = regex.compile(r'\b(%s)[%s]?$' % (small, PUNCT), regex.I)
    SUBPHRASE = regex.compile(r'([:.;?!][ ])(%s)' % small)

def retrieve_abbreviations(path_to_config=None):
    """
    This function checks for a default list of abbreviations which need to
    remain as they are (e.g. uppercase only or mixed case).
    The file is retrieved from ~/.titlecase.txt (platform independent)
    """
    logger = logging.getLogger(__name__)
    if path_to_config is None:
        path_to_config = pathlib.Path.home() / ".titlecase.txt"
    if not os.path.isfile(path_to_config):
        logger.debug('No config file found at ' + str(path_to_config))
        return lambda word, **kwargs : None
    with open(str(path_to_config)) as f:
        logger.debug('Config file used from ' + str(path_to_config))
        abbreviations = [abbr.strip() for abbr in f.read().splitlines() if abbr]
        abbreviations_capitalized = [abbr.upper() for abbr in abbreviations]
        for abbr in abbreviations:
            logging.debug("This acronym will be kept as written here: " + abbr)
        return lambda word, **kwargs : (abbreviations[abbreviations_capitalized.index(word.upper())]
                                       if word.upper() in abbreviations_capitalized else None)

def titlecase(text, callback=None, small_first_last=True):
    """
    :param text: Titlecases input text
    :param callback: Callback function that returns the titlecase version of a specific word
    :param small_first_last: Capitalize small words (e.g. 'A') at the beginning; disabled when recursing
    :type text: str
    :type callback: function
    :type small_first_last: bool

    This filter changes all words to Title Caps, and attempts to be clever
    about *un*capitalizing SMALL words like a/an/the in the input.

    The list of "SMALL words" which are not capped comes from
    the New York Times Manual of Style, plus 'vs' and 'v'.

    """

    lines = regex.split('[\r\n]+', text)
    processed = []
    for line in lines:
        all_caps = line.upper() == line
        words = regex.split('[\t ]', line)
        tc_line = []
        for word in words:
            if callback:
                new_word = callback(word, all_caps=all_caps)
                if new_word:
                    # Address #22: If a callback has done something
                    # specific, leave this string alone from now on
                    tc_line.append(_mark_immutable(new_word))
                    continue

            if all_caps:
                if UC_INITIALS.match(word):
                    tc_line.append(word)
                    continue

            if APOS_SECOND.match(word):
                if len(word[0]) == 1 and word[0] not in 'aeiouAEIOU':
                    word = word[0].lower() + word[1] + word[2].upper() + word[3:]
                else:
                    word = word[0].upper() + word[1] + word[2].upper() + word[3:]
                tc_line.append(word)
                continue

            match = MAC_MC.match(word)
            if match:
                tc_line.append("%s%s" % (match.group(1).capitalize(),
                                         titlecase(match.group(2), callback, small_first_last)))
                continue

            if INLINE_PERIOD.search(word) or (not all_caps and UC_ELSEWHERE.match(word)):
                tc_line.append(word)
                continue
            if SMALL_WORDS.match(word):
                tc_line.append(word.lower())
                continue

            if "/" in word and "//" not in word:
                slashed = map(
                    lambda t: titlecase(t,callback,False),
                    word.split('/')
                )
                tc_line.append("/".join(slashed))
                continue

            if '-' in word:
                hyphenated = map(
                    lambda t: titlecase(t, callback, small_first_last),
                    word.split('-')
                )
                tc_line.append("-".join(hyphenated))
                continue

            if all_caps:
                word = word.lower()

            # A term with all consonants should be considered an acronym.  But if it's
            # too short (like "St", don't apply this)
            CONSONANTS = ''.join(set(string.ascii_lowercase)
                                 - {'a', 'e', 'i', 'o', 'u', 'y'})
            is_all_consonants = regex.search('\A[' + CONSONANTS + ']+\Z', word,
                                             flags=regex.IGNORECASE)
            if is_all_consonants and len(word) > 2:
                tc_line.append(word.upper())
                continue

            # Just a normal word that needs to be capitalized
            tc_line.append(CAPFIRST.sub(lambda m: m.group(0).upper(), word))

        if small_first_last and tc_line:
            if not isinstance(tc_line[0], Immutable):
                tc_line[0] = SMALL_FIRST.sub(lambda m: '%s%s' % (
                    m.group(1),
                    m.group(2).capitalize()
                ), tc_line[0])

            if not isinstance(tc_line[-1], Immutable):
                tc_line[-1] = SMALL_LAST.sub(
                    lambda m: m.group(0).capitalize(), tc_line[-1]
                )

        result = " ".join(tc_line)

        result = SUBPHRASE.sub(lambda m: '%s%s' % (
            m.group(1),
            m.group(2).capitalize()
        ), result)

        processed.append(result)

    return "\n".join(processed)

def cmd():
    '''Handler for command line invocation'''

    # Try to handle any reasonable thing thrown at this.
    # Consume '-f' and '-o' as input/output, allow '-' for stdin/stdout
    # and treat any subsequent arguments as a space separated string to
    # be titlecased (so it still works if people forget quotes)
    parser = argparse.ArgumentParser()
    in_group = parser.add_mutually_exclusive_group()
    in_group.add_argument('string', nargs='*', default=[],
            help='String to titlecase')
    in_group.add_argument('-f', '--input-file',
            help='File to read from to titlecase')
    parser.add_argument('-o', '--output-file',
            help='File to write titlecased output to')

    args = parser.parse_args()

    if args.input_file is not None:
        if args.input_file == '-':
            ifile = sys.stdin
        else:
            ifile = open(args.input_file)
    else:
        ifile = sys.stdin

    if args.output_file is not None:
        if args.output_file == '-':
            ofile = sys.stdout
        else:
            ofile = open(args.output_file, 'w')
    else:
        ofile = sys.stdout

    if len(args.string) > 0:
        in_string = ' '.join(args.string)
    else:
        with ifile:
            in_string = ifile.read()

    with ofile:
        ofile.write(titlecase(in_string, callback=retrieve_abbreviations()))
<commit_msg>add wordlist<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Original Perl version by: John Gruber http://daringfireball.net/ 10 May 2008
Python version by Stuart Colville http://muffinresearch.co.uk
License: http://www.opensource.org/licenses/mit-license.php
"""

import argparse
import logging
import os
import pathlib
import re
import string
import sys

import regex

__all__ = ['titlecase']
__version__ = '1.0.0'

SMALL = r'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?'
PUNCT = r"""!"“#$%&'‘()*+,\-–‒—―./:;?@[\\\]_`{|}~"""

SMALL_WORDS = regex.compile(r'^(%s)$' % SMALL, regex.I)
INLINE_PERIOD = regex.compile(r'[\p{Letter}][.][\p{Letter}]', regex.I)
UC_ELSEWHERE = regex.compile(r'[%s]*?[\p{Letter}]+[\p{Uppercase_Letter}]+?' % PUNCT)
CAPFIRST = regex.compile(r"^[%s]*?([\p{Letter}])" % PUNCT)
SMALL_FIRST = regex.compile(r'^([%s]*)(%s)\b' % (PUNCT, SMALL), regex.I)
SMALL_LAST = regex.compile(r'\b(%s)[%s]?$' % (SMALL, PUNCT), regex.I)
SUBPHRASE = regex.compile(r'([:.;?!\-–‒—―][ ])(%s)' % SMALL)
APOS_SECOND = regex.compile(r"^[dol]{1}['‘]{1}[\p{Letter}]+(?:['s]{2})?$", regex.I)
UC_INITIALS = regex.compile(r"^(?:[\p{Uppercase_Letter}]{1}\.{1}|[\p{Uppercase_Letter}]{1}\.{1}[\p{Uppercase_Letter}]{1})+$")
MAC_MC = regex.compile(r"^([Mm]c|MC)(\w.+)")

class Immutable(object):
    pass

class ImmutableString(str, Immutable):
    pass

class ImmutableBytes(bytes, Immutable):
    pass

def _mark_immutable(text):
    if isinstance(text, bytes):
        return ImmutableBytes(text)
    return ImmutableString(text)

def set_small_word_list(small=SMALL):
    global SMALL_WORDS
    global SMALL_FIRST
    global SMALL_LAST
    global SUBPHRASE
    SMALL_WORDS = regex.compile(r'^(%s)$' % small, regex.I)
    SMALL_FIRST = regex.compile(r'^([%s]*)(%s)\b' % (PUNCT, small), regex.I)
    SMALL_LAST = regex.compile(r'\b(%s)[%s]?$' % (small, PUNCT), regex.I)
    SUBPHRASE = regex.compile(r'([:.;?!][ ])(%s)' % small)

def retrieve_abbreviations(path_to_config=None):
    """
    This function checks for a default list of abbreviations which need to
    remain as they are (e.g. uppercase only or mixed case).
    The file is retrieved from ~/.titlecase.txt (platform independent)
    """
    logger = logging.getLogger(__name__)
    if path_to_config is None:
        path_to_config = pathlib.Path.home() / ".titlecase.txt"
    if not os.path.isfile(path_to_config):
        logger.debug('No config file found at ' + str(path_to_config))
        return lambda word, **kwargs : None
    with open(str(path_to_config)) as f:
        logger.debug('Config file used from ' + str(path_to_config))
        abbreviations = [abbr.strip() for abbr in f.read().splitlines() if abbr]
        abbreviations_capitalized = [abbr.upper() for abbr in abbreviations]
        for abbr in abbreviations:
            logging.debug("This acronym will be kept as written here: " + abbr)
        return lambda word, **kwargs : (abbreviations[abbreviations_capitalized.index(word.upper())]
                                       if word.upper() in abbreviations_capitalized else None)

def titlecase(text, callback=None, small_first_last=True):
    """
    :param text: Titlecases input text
    :param callback: Callback function that returns the titlecase version of a specific word
    :param small_first_last: Capitalize small words (e.g. 'A') at the beginning; disabled when recursing
    :type text: str
    :type callback: function
    :type small_first_last: bool

    This filter changes all words to Title Caps, and attempts to be clever
    about *un*capitalizing SMALL words like a/an/the in the input.

    The list of "SMALL words" which are not capped comes from
    the New York Times Manual of Style, plus 'vs' and 'v'.

    """

    lines = regex.split('[\r\n]+', text)
    processed = []
    for line in lines:
        all_caps = line.upper() == line
        words = regex.split('[\t ]', line)
        tc_line = []
        for word in words:
            if callback:
                new_word = callback(word, all_caps=all_caps)
                if new_word:
                    # Address #22: If a callback has done something
                    # specific, leave this string alone from now on
                    tc_line.append(_mark_immutable(new_word))
                    continue

            if all_caps:
                if UC_INITIALS.match(word):
                    tc_line.append(word)
                    continue

            if APOS_SECOND.match(word):
                if len(word[0]) == 1 and word[0] not in 'aeiouAEIOU':
                    word = word[0].lower() + word[1] + word[2].upper() + word[3:]
                else:
                    word = word[0].upper() + word[1] + word[2].upper() + word[3:]
                tc_line.append(word)
                continue

            match = MAC_MC.match(word)
            if match:
                tc_line.append("%s%s" % (match.group(1).capitalize(),
                                         titlecase(match.group(2), callback, small_first_last)))
                continue

            if INLINE_PERIOD.search(word) or (not all_caps and UC_ELSEWHERE.match(word)):
                tc_line.append(word)
                continue
            if SMALL_WORDS.match(word):
                tc_line.append(word.lower())
                continue

            if "/" in word and "//" not in word:
                slashed = map(
                    lambda t: titlecase(t,callback,False),
                    word.split('/')
                )
                tc_line.append("/".join(slashed))
                continue

            if '-' in word:
                hyphenated = map(
                    lambda t: titlecase(t, callback, small_first_last),
                    word.split('-')
                )
                tc_line.append("-".join(hyphenated))
                continue

            if all_caps:
                word = word.lower()

            # A term with all consonants should be considered an acronym.  But if it's
            # too short (like "St", don't apply this)
            CONSONANTS = ''.join(set(string.ascii_lowercase)
                                 - {'a', 'e', 'i', 'o', 'u', 'y'})
            is_all_consonants = regex.search('\A[' + CONSONANTS + ']+\Z', word,
                                             flags=regex.IGNORECASE)
            if is_all_consonants and len(word) > 2:
                tc_line.append(word.upper())
                continue

            # Just a normal word that needs to be capitalized
            tc_line.append(CAPFIRST.sub(lambda m: m.group(0).upper(), word))

        if small_first_last and tc_line:
            if not isinstance(tc_line[0], Immutable):
                tc_line[0] = SMALL_FIRST.sub(lambda m: '%s%s' % (
                    m.group(1),
                    m.group(2).capitalize()
                ), tc_line[0])

            if not isinstance(tc_line[-1], Immutable):
                tc_line[-1] = SMALL_LAST.sub(
                    lambda m: m.group(0).capitalize(), tc_line[-1]
                )

        result = " ".join(tc_line)

        result = SUBPHRASE.sub(lambda m: '%s%s' % (
            m.group(1),
            m.group(2).capitalize()
        ), result)

        processed.append(result)

    return "\n".join(processed)

def cmd():
    '''Handler for command line invocation'''

    # Try to handle any reasonable thing thrown at this.
    # Consume '-f' and '-o' as input/output, allow '-' for stdin/stdout
    # and treat any subsequent arguments as a space separated string to
    # be titlecased (so it still works if people forget quotes)
    parser = argparse.ArgumentParser()
    in_group = parser.add_mutually_exclusive_group()
    in_group.add_argument('string', nargs='*', default=[],
            help='String to titlecase')
    in_group.add_argument('-f', '--input-file',
            help='File to read from to titlecase')
    parser.add_argument('-o', '--output-file',
            help='File to write titlecased output to')
    parser.add_argument('-w', '--wordlist',
            help='Wordlist for acronyms')

    args = parser.parse_args()

    if args.input_file is not None:
        if args.input_file == '-':
            ifile = sys.stdin
        else:
            ifile = open(args.input_file)
    else:
        ifile = sys.stdin

    if args.output_file is not None:
        if args.output_file == '-':
            ofile = sys.stdout
        else:
            ofile = open(args.output_file, 'w')
    else:
        ofile = sys.stdout

    if len(args.string) > 0:
        in_string = ' '.join(args.string)
    else:
        with ifile:
            in_string = ifile.read()

    with ofile:
        if args.wordlist is None:
            ofile.write(titlecase(in_string, callback=retrieve_abbreviations()))
        else:
            ofile.write(titlecase(in_string, callback=retrieve_abbreviations(args.wordlist)))
<|endoftext|>


<commit_before>#!/usr/local/bin/python
# -*- coding: utf-8 -*-

'''
Data structures that classify word types
'''

__author__ = "Rob Rayborn"
__copyright__ = "Copyright 2014, The Mozilla Foundation"
__license__ = "MPLv2"
__maintainer__ = "Rob Rayborn"
__email__ = "rrayborn@mozilla.com"
__status__ = "Development"

#TODO(rrayborn): Better URL parsing
#TODO(rrayborn): Better helpfulness

# import external language library
import csv
import re
import string
import unicodedata

from collections import defaultdict
from os import path
from lib.language.similarity import Simplifier

# globals
_PATH = path.dirname(path.realpath(__file__))+'/'
_SPAM_WORDS_CSV    = _PATH + "spam_words.csv"
_COMMON_WORDS_CSV  = _PATH + "common_words.csv"
_WORD_MAPPINGS_CSV = _PATH + "word_mappings.csv"
_TLD_CSV           = _PATH + "tlds.csv"

_DEFAULT_CLASSIFIER = None
_SIMPLIFIER         = Simplifier()

_HELPFULNESS_NUM_WORDS = 7

def tokenize(comment, word_classifier = None):
    '''
    Takes a comment in str or unicode, returns dict of stemmed words and realwords as
    str object

    Args:
        comment (string/unicode):         The raw comment string
        word_classifier (WordClassifier): The processor object, this is only
            used if you want to use non-default classifier CSV _parse_comment_with_regexes
            [Default: WordClassifier()]

    Returns:
        words_dict(defaultdict(set)): {<simplified word>: (<original words)}
        helfulness (int):            A score of the comment's helfulness

    '''
    # Instantiate classifier
    if not word_classifier:
        global _DEFAULT_CLASSIFIER
        if not _DEFAULT_CLASSIFIER:
            _DEFAULT_CLASSIFIER = WordClassifier()
        word_classifier = _DEFAULT_CLASSIFIER

    # Standardize encoding
    if isinstance(comment, str):
        comment = comment.decode('utf-8')
    elif (not isinstance(comment, unicode)):
        raise TypeError("comment should be str or unicode. Got " + type(comment))
    comment = ''.join(c for c in unicodedata.normalize('NFD', comment) \
                      if unicodedata.category(c) != 'Mn')

    # Parse
    return word_classifier.parse_comment(comment)

class WordClassifier(object):
    '''
    An object for classifying comments.  ONLY USED IF YOU WANT TO USE N
    ON-DEFAULT CSV PATHS TO POPULATE THE VALUES.  This is unlikely, but a nice
    to have option.
    '''
    def __init__(self,
                 spam_csv    = _SPAM_WORDS_CSV,
                 common_csv  = _COMMON_WORDS_CSV,
                 mapping_csv = _WORD_MAPPINGS_CSV):

        self.spam_words      = self._parse_csv(spam_csv)
        self.common_words    = self._parse_csv(common_csv)
        self.word_mappings   = {}
        self.phrase_mappings = {}
        self._load_mapping_csv(mapping_csv)

    def is_spam(self, word):
        return word in self.spam_words

    def is_common(self, word):
        return word in self.common_words

    def translate_word(self, word):
        return self.word_mappings[word] if (word in self.word_mappings.keys()) else word

    def parse_comment(self, comment):
        '''
        Parses a comment to it's components/helpfulness

        Args:
            comment (string): The raw comment string

        Returns:
            words_dict(defaultdict(set)): {<simplified word>: (<original words)}
            helfulness (int):             A score of the comment's helfulness

        '''
        words_dict  = defaultdict(set)

        comment = comment.lower()
        comment = ' ' + comment + ' '

        # Parse the phrases out
        for phrase, resolution in self.phrase_mappings.iteritems():
            if phrase in comment:
                regex = re.compile(r'([\W]+)(%s)([\W^]+)' % phrase)
                regex_iter = regex.finditer(comment)
                i = 0
                for match in regex_iter:
                    current_resolution = match.group(1) + resolution + match.group(3)
                    comment = comment[:match.start() - i] + current_resolution + comment[match.end() - i:]
                    len_old = len(match.group(0))
                    len_new = len(current_resolution)
                    i = len_old - len_new
        comment = comment[1:-1]

        # Regex Matches
        comment = self._parse_comment_with_regexes(comment, words_dict)

        # Tokenize
        words = re.split(r"[|,]|\s+|[^\w'.-]+|[-.'](\s|$)", comment.encode('utf-8'))

        num_words = len(words)/2
        helpfulness = 10 if num_words > _HELPFULNESS_NUM_WORDS else 6
        for word in words:

            helpfulness = min(self._parse_word(word, words_dict = words_dict), helpfulness)

        return words_dict, helpfulness

    def _parse_word(self, word, words_dict = None, original = None):
        '''
        Parses a word to it's helpfulness rating and simplified value. Has a
        side-effect of updating the words_dict.

        Args:
            word (string):                The raw word
            words_dict(defaultdict(set)): The words_dict to update
            original (string):            Overrides the key value to update in words_dict

        Returns:
            helfulness (int): A score of the comment's helfulness
            word (str):       The simplified word value

        '''
        simplifier = _SIMPLIFIER
        nonword_regex = re.compile(r'\W+')
        if not original:
            original = word
        helpfulness = 10

        if not word or nonword_regex.match(word): #TODO(rrayborn): this shouldn't happen but it is
            return helpfulness, None

        # Apply Mappings
        word = self.translate_word(word)
        # Remove common [Pre]
        if self.is_common(word):
            return helpfulness, None
        # Check for spam
        if self.is_spam(word):
            helpfulness = 0
            return helpfulness, None
        # Stem
        word = simplifier.simplify(word)
        # Remove common [Post]
        if self.is_common(word):
            return helpfulness, None
        if words_dict is not None:
            words_dict[word].add(original)
        return helpfulness, word

    # === FILE LOADERS =========================================================

    def _load_mapping_csv(self, filename = _WORD_MAPPINGS_CSV):
        with open(filename, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=',', quotechar='\"')
            for row in csv_reader:
                if len(row) == 2:
                    self.phrase_mappings[row[0]] = row[1]
                    original = row[0]
                    new = row[1]
                    delim_re = re.compile('[\W_ ]+')
                    if delim_re.search(original):
                        self.phrase_mappings[original] = new
                    else:
                        self.word_mappings[original]   = new
                else:
                    raise Warning("File %s does not have 2 columns as required" % filename)

    def _parse_csv(self, filename):
        ret = set()
        with open(filename, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=',', quotechar='\"')
            for row in csv_reader:
                if len(row) == 1:
                    ret.add(row[0])
                else:
                    raise Warning("File %s does not have 1 column as required" % filename)
        return ret

    # === REGEX-BASED PARSING ==================================================

    def _get_url_regex(self, filename = _TLD_CSV):
        '''Returns a compiled URL regex with domains as specified in <filename>'''

        # create regex
        base_regex = r'(?:http[s]?://)?([a-zA-Z0-9]*\.)*([a-zA-Z0-9]+)\.(?:%s)+(?:\/(?:[^ ])*)?' #

        tlds = []
        with open(filename, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=',', quotechar='\"')
            for row in csv_reader:
                if len(row) == 1:
                    tlds.append(row[0].replace('.','\\.'))
                else:
                    raise Warning("File %s does not have 1 column as required" % filename)

        return re.compile(base_regex % '|'.join(tlds))

    def _get_version_regex(self):
        '''Returns a complied version regex'''
        regex = r'(?:firefox|version|release|firefox version|firefox release)[\W]{1,3}[0-9]{1,3}'
        return re.compile(regex)

    def _parse_comment_with_regexes(self, comment, words_dict):
        '''Applies all of our regexes to parse the comment'''
        comment = self._parse_comment_with_regex(
                comment, self._get_url_regex(), words_dict = words_dict
            )
        comment = self._parse_comment_with_regex(
                comment, self._get_version_regex(), replacement = ''
            )

        return comment

    def _parse_comment_with_regex(self, comment, regex, words_dict = None, replacement = None):
        '''Applies a regex to parse the comment'''
        match = regex.search(comment)
        if match:
            # print ['match',match.group(0)]
            if replacement is None:
                ignore, rep = self._parse_word(match.group(1))
            else:
                rep = replacement
            if words_dict is not None:
                words_dict[rep].add(match.group(0))
                comment = comment[:match.start()] + '' + \
                        self._parse_comment_with_regex(
                                comment[match.end():], regex,
                                words_dict = words_dict, replacement = replacement
                            )
            else:
                comment = comment[:match.start()] + ' ' + rep + ' ' + \
                        self._parse_comment_with_regex(
                                comment[match.end():], regex,
                                words_dict = words_dict, replacement = replacement
                            )

        return comment

#def main():
##    _word_classifier = WordClassifier()
##    print ["_word_classifier.is_spam('palemoon')          ", _word_classifier.is_spam('palemoon')]
##    print ["_word_classifier.is_spam('palemoom')          ", _word_classifier.is_spam('palemoom')]
##    print ["_word_classifier.is_common('a')               ", _word_classifier.is_common('a')]
##    print ["_word_classifier.is_common('apple')           ", _word_classifier.is_common('apple')]
##    print ["_word_classifier.translate_word('ff')      ", _word_classifier.translate_word('ff')]
##    print ["_word_classifier.translate_word('firefox') ", _word_classifier.translate_word('firefox')]
##    print ["_word_classifier.translate_word('cats')    ", _word_classifier.translate_word('cats')]
#    examples = [
#            ' Fire fox 35 is crashing on http://a.b.c.youtube.com . I don\'t  like when fire fox crashes on youtube.com. Tést',
#            ' Stack Overflow is a question and answer site for professional and enthusiast programmers. It\'s 100 free, no registration required.',
#            'fire fox crash'
#        ]
#    for example in examples:
#        print example
#        a,b = tokenize(example)
#        print a
#        print b
#
#
#if __name__ == '__main__':
#    main()
<commit_msg>Fix URL parsing for subdomains<commit_after>#!/usr/local/bin/python
# -*- coding: utf-8 -*-

'''
Data structures that classify word types
'''

__author__ = "Rob Rayborn"
__copyright__ = "Copyright 2014, The Mozilla Foundation"
__license__ = "MPLv2"
__maintainer__ = "Rob Rayborn"
__email__ = "rrayborn@mozilla.com"
__status__ = "Development"

#TODO(rrayborn): Better helpfulness

# import external language library
import csv
import re
import string
import unicodedata

from collections import defaultdict
from os import path
from lib.language.similarity import Simplifier

# globals
_PATH = path.dirname(path.realpath(__file__))+'/'
_SPAM_WORDS_CSV    = _PATH + "spam_words.csv"
_COMMON_WORDS_CSV  = _PATH + "common_words.csv"
_WORD_MAPPINGS_CSV = _PATH + "word_mappings.csv"
_TLD_CSV           = _PATH + "tlds.csv"

_DEFAULT_CLASSIFIER = None
_SIMPLIFIER         = Simplifier()

_HELPFULNESS_NUM_WORDS = 7

def tokenize(comment, word_classifier = None):
    '''
    Takes a comment in str or unicode, returns dict of stemmed words and realwords as
    str object

    Args:
        comment (string/unicode):         The raw comment string
        word_classifier (WordClassifier): The processor object, this is only
            used if you want to use non-default classifier CSV _parse_comment_with_regexes
            [Default: WordClassifier()]

    Returns:
        words_dict(defaultdict(set)): {<simplified word>: (<original words)}
        helfulness (int):            A score of the comment's helfulness

    '''
    # Instantiate classifier
    if not word_classifier:
        global _DEFAULT_CLASSIFIER
        if not _DEFAULT_CLASSIFIER:
            _DEFAULT_CLASSIFIER = WordClassifier()
        word_classifier = _DEFAULT_CLASSIFIER

    # Standardize encoding
    if isinstance(comment, str):
        comment = comment.decode('utf-8')
    elif (not isinstance(comment, unicode)):
        raise TypeError("comment should be str or unicode. Got " + type(comment))
    comment = ''.join(c for c in unicodedata.normalize('NFD', comment) \
                      if unicodedata.category(c) != 'Mn')

    # Parse
    return word_classifier.parse_comment(comment)

class WordClassifier(object):
    '''
    An object for classifying comments.  ONLY USED IF YOU WANT TO USE N
    ON-DEFAULT CSV PATHS TO POPULATE THE VALUES.  This is unlikely, but a nice
    to have option.
    '''
    def __init__(self,
                 spam_csv    = _SPAM_WORDS_CSV,
                 common_csv  = _COMMON_WORDS_CSV,
                 mapping_csv = _WORD_MAPPINGS_CSV):

        self.spam_words      = self._parse_csv(spam_csv)
        self.common_words    = self._parse_csv(common_csv)
        self.word_mappings   = {}
        self.phrase_mappings = {}
        self._load_mapping_csv(mapping_csv)

    def is_spam(self, word):
        return word in self.spam_words

    def is_common(self, word):
        return word in self.common_words

    def translate_word(self, word):
        return self.word_mappings[word] if (word in self.word_mappings.keys()) else word

    def parse_comment(self, comment):
        '''
        Parses a comment to it's components/helpfulness

        Args:
            comment (string): The raw comment string

        Returns:
            words_dict(defaultdict(set)): {<simplified word>: (<original words)}
            helfulness (int):             A score of the comment's helfulness

        '''
        words_dict  = defaultdict(set)

        comment = comment.lower()
        comment = ' ' + comment + ' '

        # Parse the phrases out
        for phrase, resolution in self.phrase_mappings.iteritems():
            if phrase in comment:
                regex = re.compile(r'([\W]+)(%s)([\W^]+)' % phrase)
                regex_iter = regex.finditer(comment)
                i = 0
                for match in regex_iter:
                    current_resolution = match.group(1) + resolution + match.group(3)
                    comment = comment[:match.start() - i] + current_resolution + comment[match.end() - i:]
                    len_old = len(match.group(0))
                    len_new = len(current_resolution)
                    i = len_old - len_new
        comment = comment[1:-1]

        # Regex Matches
        comment = self._parse_comment_with_regexes(comment, words_dict)

        # Tokenize
        words = re.split(r"[|,]|\s+|[^\w'.-]+|[-.'](\s|$)", comment.encode('utf-8'))

        num_words = len(words)/2
        helpfulness = 10 if num_words > _HELPFULNESS_NUM_WORDS else 6
        for word in words:

            helpfulness = min(self._parse_word(word, words_dict = words_dict), helpfulness)

        return words_dict, helpfulness

    def _parse_word(self, word, words_dict = None, original = None):
        '''
        Parses a word to it's helpfulness rating and simplified value. Has a
        side-effect of updating the words_dict.

        Args:
            word (string):                The raw word
            words_dict(defaultdict(set)): The words_dict to update
            original (string):            Overrides the key value to update in words_dict

        Returns:
            helfulness (int): A score of the comment's helfulness
            word (str):       The simplified word value

        '''
        simplifier = _SIMPLIFIER
        nonword_regex = re.compile(r'\W+')
        if not original:
            original = word
        helpfulness = 10

        if not word or nonword_regex.match(word): #TODO(rrayborn): this shouldn't happen but it is
            return helpfulness, None

        # Apply Mappings
        word = self.translate_word(word)
        # Remove common [Pre]
        if self.is_common(word):
            return helpfulness, None
        # Check for spam
        if self.is_spam(word):
            helpfulness = 0
            return helpfulness, None
        # Stem
        word = simplifier.simplify(word)
        # Remove common [Post]
        if self.is_common(word):
            return helpfulness, None
        if words_dict is not None:
            words_dict[word].add(original)
        return helpfulness, word

    # === FILE LOADERS =========================================================

    def _load_mapping_csv(self, filename = _WORD_MAPPINGS_CSV):
        with open(filename, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=',', quotechar='\"')
            for row in csv_reader:
                if len(row) == 2:
                    self.phrase_mappings[row[0]] = row[1]
                    original = row[0]
                    new = row[1]
                    delim_re = re.compile('[\W_ ]+')
                    if delim_re.search(original):
                        self.phrase_mappings[original] = new
                    else:
                        self.word_mappings[original]   = new
                else:
                    raise Warning("File %s does not have 2 columns as required" % filename)

    def _parse_csv(self, filename):
        ret = set()
        with open(filename, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=',', quotechar='\"')
            for row in csv_reader:
                if len(row) == 1:
                    ret.add(row[0])
                else:
                    raise Warning("File %s does not have 1 column as required" % filename)
        return ret

    # === REGEX-BASED PARSING ==================================================

    def _get_url_regex(self, filename = _TLD_CSV):
        '''Returns a compiled URL regex with domains as specified in <filename>'''

        # create regex
        base_regex = r'(?:http[s]?://)?([a-zA-Z0-9]*\.)*(?P<domain>[a-zA-Z0-9]+)\.(?:%s)+(?:\/(?:[^ ])*)?' #

        tlds = []
        with open(filename, 'rb') as csv_file:
            csv_reader = csv.reader(csv_file, delimiter=',', quotechar='\"')
            for row in csv_reader:
                if len(row) == 1:
                    tlds.append(row[0].replace('.','\\.'))
                else:
                    raise Warning("File %s does not have 1 column as required" % filename)

        return re.compile(base_regex % '|'.join(tlds))

    def _get_version_regex(self):
        '''Returns a complied version regex'''
        regex = r'(?:firefox|version|release|firefox version|firefox release)[\W]{1,3}[0-9]{1,3}'
        return re.compile(regex)

    def _parse_comment_with_regexes(self, comment, words_dict):
        '''Applies all of our regexes to parse the comment'''
        comment = self._parse_comment_with_regex(
                comment, self._get_url_regex(), words_dict = words_dict
            )
        comment = self._parse_comment_with_regex(
                comment, self._get_version_regex(), replacement = ''
            )

        return comment

    def _parse_comment_with_regex(self, comment, regex, words_dict = None, replacement = None):
        '''Applies a regex to parse the comment'''
        match = regex.search(comment)
        if match:
            # print ['match',match.group(0)]
            if replacement:
                rep = replacement
            else:
                try:
                    ignore, rep = self._parse_word(match.group('domain'))
                except Exception:
                    return comment
            if words_dict is not None:
                words_dict[rep].add(match.group(0))
                comment = comment[:match.start()] + '' + \
                        self._parse_comment_with_regex(
                                comment[match.end():], regex,
                                words_dict = words_dict, replacement = replacement
                            )
            else:
                comment = comment[:match.start()] + ' ' + rep + ' ' + \
                        self._parse_comment_with_regex(
                                comment[match.end():], regex,
                                words_dict = words_dict, replacement = replacement
                            )

        return comment

def main():
#    _word_classifier = WordClassifier()
#    print ["_word_classifier.is_spam('palemoon')          ", _word_classifier.is_spam('palemoon')]
#    print ["_word_classifier.is_spam('palemoom')          ", _word_classifier.is_spam('palemoom')]
#    print ["_word_classifier.is_common('a')               ", _word_classifier.is_common('a')]
#    print ["_word_classifier.is_common('apple')           ", _word_classifier.is_common('apple')]
#    print ["_word_classifier.translate_word('ff')      ", _word_classifier.translate_word('ff')]
#    print ["_word_classifier.translate_word('firefox') ", _word_classifier.translate_word('firefox')]
#    print ["_word_classifier.translate_word('cats')    ", _word_classifier.translate_word('cats')]
    examples = [
            ' Fire fox 35 is crashing on http://a.b.c.youtube.com . I don\'t  like when fire fox crashes on youtube.com. Tést.com',
            ' Stack Overflow is a question and answer site for professional and enthusiast programmers. It\'s 100 free, no registration required.',
            'fire fox crash',
            'For the paper called "The gravity tunnel in a non-uniform Earth" Alexander R. Klotz, when i open the pdf in firefox on page 3, the square root sign is almost touching the text below it in equation (3) in auto zoom, see (http://imgur.com/DlX1Ogw) and it shows better when set to page width see (http://imgur.com/PRC7kW2). this has been present for many releases, hope you can add it to your list of things to fix in future pdf.js releases! thanks'
        ]
    for example in examples:
        print example
        a,b = tokenize(example)
        print a
        print b

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#-*- coding: utf-8 -*-
"""
    digilentdriver
    ~~~~~~~~~~~~~~

    Digilent driver for Linux USB JTAG controller

    :copyright: (c) 2014 by Jessy Diamond Exum
    :license: Pending, see LICENSE for more details.
"""
from time import time
import math
import numbers

from proteusisc.cabledriver import CableDriver
from proteusisc.primitive import Level1Primitive,\
    Level2Primitive, Level3Primitive, Executable
from proteusisc.contracts import NOCARE, ZERO, ONE, CONSTANT, ARBITRARY
from proteusisc.errors import JTAGEnableFailedError,\
    JTAGAlreadyEnabledError, JTAGNotEnabledError
from proteusisc.bittypes import ConstantBitarray, CompositeBitarray,\
    Bitarray

#PROG = 8
#TCK = 4
#TMS = 2
#TDI = 1
#TDO = 1

class XPC1TransferPrimitive(Level1Primitive, Executable):
    #transfer_bits_single can be used for single bit jtag transfers.
    #This will be necessary for firmware upgrade.
    _function_name = 'transfer_bits'
    _driver_function_name = 'transfer_bits'#_single'#_cpld_upgrade'
    _max_bits = 65536
    _TMS, _TDI, _TDO = ARBITRARY, ARBITRARY, ARBITRARY
    def _get_args(self):
        from time import time
        t = time()
        if isinstance(self.tdo, CompositeBitarray):
            self.tdo = self.tdo.prepare(preserve_history=True)
        print("DRIVER FUNCTION ARGUMENT PREPARE TIME", time()-t)

        return [self.count], {"TMS":self.tms, "TDI":self.tdi,
                              'TDO': self.tdo}

class XilinxPC1Driver(CableDriver):
    _primitives = [XPC1TransferPrimitive]
    def __init__(self, dev):
        super(XilinxPC1Driver, self).__init__(dev)
        h = self._dev.open()

        self.serialNumber = '000000000000'
        self.name = 'PC1_'+self.serialNumber[-4:]
        self.productId = 0

        self.productName = 'Platform Cable 1'
        self.firmwareVersion = 0

        #Physically set the controller Speed
        self.speed = 12000000
        h.close()

    def __repr__(self): #pragma: no cover
        return "%s(%s; Name: %s; SN: %s; FWver: %04x)"%\
                                         (self.__class__.__name__,
                                          self.productName,
                                          self.name,
                                          self.serialNumber,
                                          self.firmwareVersion)

    def jtag_enable(self):
        if self._dev.getProductID() == 0x0D:
            self._handle.claimInterface(0)
            self._handle.setInterfaceAltSetting(0,1)
        self.xpcu_enable_output(True)
        self.xpcu_set_jtag_speed(False)
        self._jtagon = True
        #self.xpcu_enable_cpld_upgrade_mode(True)

    def jtag_disable(self):
        self.xpcu_enable_output(False)
        if self._dev.getProductID() == 0x0D and self._jtagon:
            self._handle.releaseInterface(0)
        self._jtagon = False

        #self.xpcu_enable_cpld_upgrade_mode(False)

    #@profile
    def transfer_bits(self, count, *, TMS=True, TDI=False, TDO=False):
        if not self._jtagon:
            raise JTAGNotEnabledError('JTAG Must be enabled first')
        if count < 1:
            raise ValueError()
        if count > 0xFFFFFF+1:
            raise ValueError("Too many transactions. Max 16777216.")

        if isinstance(TMS, (numbers.Number, bool)):
            TMS = ConstantBitarray(bool(TMS), count)
        if isinstance(TDI, (numbers.Number, bool)):
            TDI = ConstantBitarray(bool(TDI), count)
        if isinstance(TDO, (numbers.Number, bool)):
            TDO = ConstantBitarray(bool(TDO), count)

        #if self._scanchain:
        #    t = time()
        #    self._scanchain._tap_transition_driver_trigger(TMS)
        #    print("XPCU1 Sync State Machine Time:", time()-t)

        t = time()
        bit_return_count = TDO.count(True)
        print("BIT RETURN COUNT CALCULATION TIME", time()-t)
        print("BIT RETURN COUNT", bit_return_count, len(TDO), count)

        t = time()
        outdata = bytearray(int(math.ceil(count/4.0))*2)
        tmsbytes = TMS.tobytes()
        tdibytes = TDI.tobytes()
        tdobytes = TDO.tobytes()
        print("XPCU1 TMS/TDI/TDO Buffer Prepare Time:", time()-t)

        t = time()
        adjusted_count = math.ceil(count/4)*4
        outbaseindex = 0
        inoffset = 0
        if count%8-4>0:
            #0xAa 0xBb 0xCc 0xDd = 0xab 0xcd
            outdata[0], outdata[1] = \
                ((tmsbytes[-1]<<4)&0xF0)|(tdibytes[-1]&0xF), \
                ((tdobytes[-1]<<4)&0xF0)|(0xF<<(4-(count%4)))&0xF
            outbaseindex = 2
        if count%8:
            #0xAa 0xBb 0xCc 0xDd = 0xAB 0xCD
            outdata[outbaseindex], outdata[outbaseindex+1] = \
                (tmsbytes[-1]&0xF0)|(tdibytes[-1]>>4), \
                (tdobytes[-1]&0xF0)|(0xFF<<(4-min(4, count%8)))&0xF
            outbaseindex += 2
            inoffset = 1

        readoffset = -(inoffset+1)
        # This is done this way because breaking these into variables
        # blows up the runtime. Thanks to mekarpeles for finding this.
        # Bit shifts and readoffset increased performance slightly.
        # Encoding 16777216 bits takes 3.2s, down from 80s (on 2.9 GHZ i7-3520M)
        for i in range(len(tmsbytes)-inoffset):#range(len(outdata)//4):
            outdata[(i<<2)+outbaseindex], outdata[(i<<2)+1+outbaseindex], \
                outdata[(i<<2)+2+outbaseindex], outdata[(i<<2)+3+outbaseindex] \
                = \
                ((tmsbytes[readoffset-i]&0x0F)<<4)|(tdibytes[readoffset-i]&0x0F), \
                ((tdobytes[readoffset-i]&0x0F)<<4)|0x0F,\
                (tmsbytes[readoffset-i]&0xF0)|(tdibytes[readoffset-i]>>4), \
                (tdobytes[readoffset-i]&0xF0)|0x0F

        print("XPCU1 byte blocks 2 Data Prepare Time:", time()-t)

        #print("LENGTH OF OUTDATA", len(outdata))
        return self.xpcu_GPIO_transfer(adjusted_count, outdata,
                    bit_return_count=bit_return_count)

    def transfer_bits_single(self, count, TMS, TDI, TDO=False):
        if not self._jtagon:
            raise JTAGNotEnabledError()
        if isinstance(TMS, (numbers.Number, bool)):
            TMS = Bitarray(count*('1' if TMS else '0'))
        if isinstance(TDI, (numbers.Number, bool)):
            TDI = Bitarray(count*('1' if TDI else '0'))
        #if isinstance(TDO, (numbers.Number, bool)):
        #    TDO = Bitarray(count*('1' if TDO else '0'))
        if self._scanchain:
            self._scanchain._tap_transition_driver_trigger(TMS)
        #self.xpcu_single_read()
        outbits = Bitarray()
        TMS.reverse()
        TDI.reverse()

        for bit_num in range(count):
            self.xpcu_single_write(TMS[bit_num], TDI[bit_num])
            if TDO:
                b = self.xpcu_single_read()
                outbits.append(b)

        if outbits:
            outbits.reverse()
            #print(outbits, len(outbits))
            return outbits

    def transfer_bits_single_cpld_upgrade(self, count, TMS, TDI, TDO=False):
        if not self._jtagon:
            raise JTAGNotEnabledError()
        if isinstance(TMS, (numbers.Number, bool)):
            TMS = Bitarray(count*('1' if TMS else '0'))
        if isinstance(TDI, (numbers.Number, bool)):
            TDI = Bitarray(count*('1' if TDI else '0'))
        #if isinstance(TDO, (numbers.Number, bool)):
        #    TDO = Bitarray(count*('1' if TDO else '0'))
        if self._scanchain:
            self._scanchain._tap_transition_driver_trigger(TMS)
        #self.xpcu_single_read()
        outbits = Bitarray()
        TMS.reverse()
        TDI.reverse()

        for bit_num in range(count):
            if TDO:
                b = self.xpcu_single_read()
                outbits.append(b)
            self.xpcu_single_write(TMS[bit_num], TDI[bit_num])

        if outbits:
            outbits.reverse()
            print(outbits, len(outbits))
            return outbits

    def xpcu_enable_output(self, enable):
        self._handle.controlWrite(0x40, 0xb0, 0x18 if enable else 0x10, 0, b'')

    def xpcu_enable_cpld_upgrade_mode(self, enable):
        self._handle.controlWrite(0x40, 0xb0, 0x52, 1 if enable else 0, b'')

    def xpcu_set_jtag_speed(self, speed_mode):
        self._handle.controlWrite(0x40, 0xb0, 0x28, 0x10|speed_mode, b'')

    def xpcu_get_GPIO_state(self):
        return ord(self._handle.controlRead(0xc0, 0xb0, 0x38, 0, 1))

    def xpcu_single_write(self, TMS, TDI):
        val = 0b100|(TMS<<1)|TDI
        self._handle.controlWrite(0x40, 0xb0, 0x30, val, b'')
        val = (TMS<<1)|TDI
        self._handle.controlWrite(0x40, 0xb0, 0x30, val, b'')

    def xpcu_single_read(self):
        b = self._handle.controlRead(0xC0, 0xb0, 0x38, 0, 1)
        #print(bin(ord(b)))
        return bool(ord(b)&1)

    def xpcu_GPIO_transfer(self, bit_count, data, *, bit_return_count=None):
        if bit_count < 1:
            raise ValueError()
        if bit_count > 0xFFFFFF+1:
            raise ValueError("Too many transactions. Max 16777216.")

        bit_count_dev = bit_count-1 #Controller uses 0 based bitcount.
        bit_count_low = bit_count_dev & 0xFFFF #16 bits for 'index' field
        bit_count_high = (bit_count_dev>>8) & 0xFF00

        from time import time
        t = time()
        if self._scanchain and self._scanchain._debug:
            print("***INPUT DATA TO CPXU (%s bits):"%(bit_count),
                  " ".join((hex(data)[2:].zfill(2)for data in data)))
        if bit_return_count is None:
            bit_return_count = XilinxPC1Driver._count_tdo_bits(data,
                                                               bit_count)
        print("COUNT TDO BITS time        ", time()-t)

        #print("VALUE", hex(bit_count_high | 0xa6)[2:].zfill(4),
        #      "INDEX", hex(bit_count_low)[2:].zfill(4),
        #      "DATLEN", len(data))

        self._handle.controlWrite(0x40, 0xb0, bit_count_high | 0xa6,
                                  bit_count_low, b'')

        #print("DATA OUT", data)

        t = time()
        bytec = self._handle.bulkWrite(2, data, timeout=120000)
        print("TRANSFER time              ", time()-t)

        if bit_return_count:
            t = time()
            bytes_wanted = int(math.ceil(bit_return_count/8.0))
            bytes_expected = bytes_wanted +(1 if bytes_wanted%2 else 0)
            print("WANTED %s; EXPECTED %s"%(bytes_wanted, bytes_expected))
            ret = self._handle.bulkRead(6, bytes_expected, timeout=5000)

            if len(ret) != bytes_expected:
                raise Exception("Data returned is wrong lentgh. "
                                "Expected %s; Got %s. This is likely an "
                                "issue with the controller. Please report "
                                "The data you sent caused this error."
                                %(bytes_expected, len(ret)))

            #print(ret.hex())
            if self._scanchain and self._scanchain._debug:
                print("OUTPUT DATA FROM XPCU (retbits: %s)"%bit_return_count,
                      " ".join((hex(data)[2:].zfill(2)for data in ret)))

            raw_bits = XilinxPC1Driver._decode_tdo_bits(
                ret, bit_return_count=bit_return_count)

            assert len(raw_bits) == bit_return_count, \
                "WRONG BIT NUM CALCULATED; returned: %s; expected: %s"%\
                (len(raw_bits), bit_return_count)
            print("RETURN DATA CALCULATION time", time()-t)
            return raw_bits

    def _get_speed(self):
        return self._set_speed(12000000)

    def _set_speed(self, speed):
        power = min(4,max(0,math.floor(math.log2(speed/750000))))
        self.xpcu_set_jtag_speed(4-power)
        return 750000 * (2**power)

    @staticmethod
    def _count_tdo_bits(data, bit_count):
        return bin(sum([((ord(data[i*2+1:i*2+2])>>4) &
                         (( 1<< min(4, bit_count-(i*4)) )-1) )<<4*i
                        for i in range(int(len(data)/2))])).count('1')

    @staticmethod
    def _decode_tdo_bits(ret, *, bit_return_count=None, bit_count=None):
        if bit_return_count is None:
            bit_return_count = XilinxPC1Driver._count_tdo_bits(data,
                                                               bit_count)

        bytes_wanted = int(math.ceil(bit_return_count/8.0))
        bytes_expected = bytes_wanted +(1 if bytes_wanted%2 else 0)

        final_group_index = (bit_return_count-(bit_return_count%32))//8
        retiter = iter(ret[:final_group_index])
        fullgroups = [bytes(elem[::-1]) for elem in
                      zip(retiter, retiter, retiter, retiter)][::-1]
        other=ret[final_group_index:][::-1]
        other_bits = Bitarray()
        other_bits.frombytes(other)
        other_bits = other_bits[:bit_return_count-(8*final_group_index)]

        reordered_data = b"".join(fullgroups)
        raw_bits = Bitarray()
        raw_bits.frombytes(reordered_data)
        raw_bits = other_bits + raw_bits

        return raw_bits

__filter__ = [((0x03FD, 0x000D),XilinxPC1Driver),
              ((0x03FD, 0x0008),XilinxPC1Driver)]
<commit_msg>New experiment to speed up XPCU1 processing.<commit_after>#-*- coding: utf-8 -*-
"""
    digilentdriver
    ~~~~~~~~~~~~~~

    Digilent driver for Linux USB JTAG controller

    :copyright: (c) 2014 by Jessy Diamond Exum
    :license: Pending, see LICENSE for more details.
"""
from time import time
import math
import numbers

from proteusisc.cabledriver import CableDriver
from proteusisc.primitive import Level1Primitive,\
    Level2Primitive, Level3Primitive, Executable
from proteusisc.contracts import NOCARE, ZERO, ONE, CONSTANT, ARBITRARY
from proteusisc.errors import JTAGEnableFailedError,\
    JTAGAlreadyEnabledError, JTAGNotEnabledError
from proteusisc.bittypes import ConstantBitarray, CompositeBitarray,\
    Bitarray

#PROG = 8
#TCK = 4
#TMS = 2
#TDI = 1
#TDO = 1

class XPC1TransferPrimitive(Level1Primitive, Executable):
    #transfer_bits_single can be used for single bit jtag transfers.
    #This will be necessary for firmware upgrade.
    _function_name = 'transfer_bits'
    _driver_function_name = 'transfer_bits'#_single'#_cpld_upgrade'
    _max_bits = 65536
    _TMS, _TDI, _TDO = ARBITRARY, ARBITRARY, ARBITRARY
    def _get_args(self):
        from time import time
        t = time()
        if isinstance(self.tdo, CompositeBitarray):
            self.tdo = self.tdo.prepare(preserve_history=True)
        print("DRIVER FUNCTION ARGUMENT PREPARE TIME", time()-t)

        return [self.count], {"TMS":self.tms, "TDI":self.tdi,
                              'TDO': self.tdo}

class XilinxPC1Driver(CableDriver):
    _primitives = [XPC1TransferPrimitive]
    def __init__(self, dev):
        super(XilinxPC1Driver, self).__init__(dev)
        h = self._dev.open()

        self.serialNumber = '000000000000'
        self.name = 'PC1_'+self.serialNumber[-4:]
        self.productId = 0

        self.productName = 'Platform Cable 1'
        self.firmwareVersion = 0

        #Physically set the controller Speed
        self.speed = 12000000
        h.close()

    def __repr__(self): #pragma: no cover
        return "%s(%s; Name: %s; SN: %s; FWver: %04x)"%\
                                         (self.__class__.__name__,
                                          self.productName,
                                          self.name,
                                          self.serialNumber,
                                          self.firmwareVersion)

    def jtag_enable(self):
        if self._dev.getProductID() == 0x0D:
            self._handle.claimInterface(0)
            self._handle.setInterfaceAltSetting(0,1)
        self.xpcu_enable_output(True)
        self.xpcu_set_jtag_speed(False)
        self._jtagon = True
        #self.xpcu_enable_cpld_upgrade_mode(True)

    def jtag_disable(self):
        self.xpcu_enable_output(False)
        if self._dev.getProductID() == 0x0D and self._jtagon:
            self._handle.releaseInterface(0)
        self._jtagon = False

        #self.xpcu_enable_cpld_upgrade_mode(False)

    @profile
    def transfer_bits(self, count, *, TMS=True, TDI=False, TDO=False):
        if not self._jtagon:
            raise JTAGNotEnabledError('JTAG Must be enabled first')
        if count < 1:
            raise ValueError()
        if count > 0xFFFFFF+1:
            raise ValueError("Too many transactions. Max 16777216.")

        if isinstance(TMS, (numbers.Number, bool)):
            TMS = ConstantBitarray(bool(TMS), count)
        if isinstance(TDI, (numbers.Number, bool)):
            TDI = ConstantBitarray(bool(TDI), count)
        if isinstance(TDO, (numbers.Number, bool)):
            TDO = ConstantBitarray(bool(TDO), count)

        #if self._scanchain:
        #    t = time()
        #    self._scanchain._tap_transition_driver_trigger(TMS)
        #    print("XPCU1 Sync State Machine Time:", time()-t)

        t = time()
        bit_return_count = TDO.count(True)
        print("BIT RETURN COUNT CALCULATION TIME", time()-t)
        print("BIT RETURN COUNT", bit_return_count, len(TDO), count)

        #t = time()
        outdata = bytearray(int(math.ceil(count/4.0))*2)
        #tmsbytes = TMS.tobytes()
        #tdibytes = TDI.tobytes()
        #tdobytes = TDO.tobytes()
        #print("XPCU1 TMS/TDI/TDO Buffer Prepare Time:", time()-t)

        t = time()
        itms = iter(TMS)
        itdi = iter(TDI)
        itdo = iter(TDO)
        @profile
        def get4bits(i):
            return next(i)<<3 | next(i)<<2 | next(i)<<1 | next(i)

        outoffset = 0
        if count%4:
            remainingbitscount = count%4
            outdata[-2], outdata[-1] = \
                sum(next(itms)<<(remainingbitscount-1-bitnum) for
                    bitnum in range(remainingbitscount))<<4|\
                    sum(next(itdi)<<(remainingbitscount-1-bitnum) for
                        bitnum in range(remainingbitscount)),\
                sum(next(itdo)<<(remainingbitscount-1-bitnum) for
                    bitnum in range(remainingbitscount))<<4|\
                    ((1<<remainingbitscount)-1)
            outoffset = -2

        if count%8 > 4:
            outdata[outoffset-2], outdata[outoffset-1],\
                       = (get4bits(itms)<<4)|get4bits(itdi), \
                       (get4bits(itdo)<<4)|0x0F

        offset = count//8 - 1
        # This is done this way because breaking these into variables
        # blows up the runtime. Thanks to mekarpeles for finding this
        for i in range(count//8):#range((count-(count%4))//8):
            outdata[((offset-i)<<2)+2], outdata[((offset-i)<<2)+3], \
                outdata[(offset-i)<<2], outdata[((offset-i)<<2)+1],\
                = (get4bits(itms)<<4)|get4bits(itdi), \
                (get4bits(itdo)<<4)|0x0F, \
                (get4bits(itms)<<4)|get4bits(itdi), \
                (get4bits(itdo)<<4)|0x0F

        print("XPCU1 byte blocks 3 Data Prepare Time:", time()-t)

        #print("LENGTH OF OUTDATA", len(outdata))
        return self.xpcu_GPIO_transfer(count, outdata,
                    bit_return_count=bit_return_count)

    def transfer_bits_single(self, count, TMS, TDI, TDO=False):
        if not self._jtagon:
            raise JTAGNotEnabledError()
        if isinstance(TMS, (numbers.Number, bool)):
            TMS = Bitarray(count*('1' if TMS else '0'))
        if isinstance(TDI, (numbers.Number, bool)):
            TDI = Bitarray(count*('1' if TDI else '0'))
        #if isinstance(TDO, (numbers.Number, bool)):
        #    TDO = Bitarray(count*('1' if TDO else '0'))
        if self._scanchain:
            self._scanchain._tap_transition_driver_trigger(TMS)
        #self.xpcu_single_read()
        outbits = Bitarray()
        TMS.reverse()
        TDI.reverse()

        for bit_num in range(count):
            self.xpcu_single_write(TMS[bit_num], TDI[bit_num])
            if TDO:
                b = self.xpcu_single_read()
                outbits.append(b)

        if outbits:
            outbits.reverse()
            #print(outbits, len(outbits))
            return outbits

    def transfer_bits_single_cpld_upgrade(self, count, TMS, TDI, TDO=False):
        if not self._jtagon:
            raise JTAGNotEnabledError()
        if isinstance(TMS, (numbers.Number, bool)):
            TMS = Bitarray(count*('1' if TMS else '0'))
        if isinstance(TDI, (numbers.Number, bool)):
            TDI = Bitarray(count*('1' if TDI else '0'))
        #if isinstance(TDO, (numbers.Number, bool)):
        #    TDO = Bitarray(count*('1' if TDO else '0'))
        if self._scanchain:
            self._scanchain._tap_transition_driver_trigger(TMS)
        #self.xpcu_single_read()
        outbits = Bitarray()
        TMS.reverse()
        TDI.reverse()

        for bit_num in range(count):
            if TDO:
                b = self.xpcu_single_read()
                outbits.append(b)
            self.xpcu_single_write(TMS[bit_num], TDI[bit_num])

        if outbits:
            outbits.reverse()
            print(outbits, len(outbits))
            return outbits

    def xpcu_enable_output(self, enable):
        self._handle.controlWrite(0x40, 0xb0, 0x18 if enable else 0x10, 0, b'')

    def xpcu_enable_cpld_upgrade_mode(self, enable):
        self._handle.controlWrite(0x40, 0xb0, 0x52, 1 if enable else 0, b'')

    def xpcu_set_jtag_speed(self, speed_mode):
        self._handle.controlWrite(0x40, 0xb0, 0x28, 0x10|speed_mode, b'')

    def xpcu_get_GPIO_state(self):
        return ord(self._handle.controlRead(0xc0, 0xb0, 0x38, 0, 1))

    def xpcu_single_write(self, TMS, TDI):
        val = 0b100|(TMS<<1)|TDI
        self._handle.controlWrite(0x40, 0xb0, 0x30, val, b'')
        val = (TMS<<1)|TDI
        self._handle.controlWrite(0x40, 0xb0, 0x30, val, b'')

    def xpcu_single_read(self):
        b = self._handle.controlRead(0xC0, 0xb0, 0x38, 0, 1)
        #print(bin(ord(b)))
        return bool(ord(b)&1)

    def xpcu_GPIO_transfer(self, bit_count, data, *, bit_return_count=None):
        if bit_count < 1:
            raise ValueError()
        if bit_count > 0xFFFFFF+1:
            raise ValueError("Too many transactions. Max 16777216.")

        bit_count_dev = bit_count-1 #Controller uses 0 based bitcount.
        bit_count_low = bit_count_dev & 0xFFFF #16 bits for 'index' field
        bit_count_high = (bit_count_dev>>8) & 0xFF00

        from time import time
        t = time()
        if self._scanchain and self._scanchain._debug:
            print("***INPUT DATA TO CPXU (%s bits):"%(bit_count),
                  " ".join((hex(data)[2:].zfill(2)for data in data)))
        if bit_return_count is None:
            bit_return_count = XilinxPC1Driver._count_tdo_bits(data,
                                                               bit_count)
        print("COUNT TDO BITS time        ", time()-t)

        #print("VALUE", hex(bit_count_high | 0xa6)[2:].zfill(4),
        #      "INDEX", hex(bit_count_low)[2:].zfill(4),
        #      "DATLEN", len(data))

        self._handle.controlWrite(0x40, 0xb0, bit_count_high | 0xa6,
                                  bit_count_low, b'')

        #print("DATA OUT", data)

        t = time()
        bytec = self._handle.bulkWrite(2, data, timeout=120000)
        print("TRANSFER time              ", time()-t)

        if bit_return_count:
            t = time()
            bytes_wanted = int(math.ceil(bit_return_count/8.0))
            bytes_expected = bytes_wanted +(1 if bytes_wanted%2 else 0)
            print("WANTED %s; EXPECTED %s"%(bytes_wanted, bytes_expected))
            ret = self._handle.bulkRead(6, bytes_expected, timeout=5000)

            if len(ret) != bytes_expected:
                raise Exception("Data returned is wrong lentgh. "
                                "Expected %s; Got %s. This is likely an "
                                "issue with the controller. Please report "
                                "The data you sent caused this error."
                                %(bytes_expected, len(ret)))

            #print(ret.hex())
            if self._scanchain and self._scanchain._debug:
                print("OUTPUT DATA FROM XPCU (retbits: %s)"%bit_return_count,
                      " ".join((hex(data)[2:].zfill(2)for data in ret)))

            raw_bits = XilinxPC1Driver._decode_tdo_bits(
                ret, bit_return_count=bit_return_count)

            assert len(raw_bits) == bit_return_count, \
                "WRONG BIT NUM CALCULATED; returned: %s; expected: %s"%\
                (len(raw_bits), bit_return_count)
            print("RETURN DATA CALCULATION time", time()-t)
            return raw_bits

    def _get_speed(self):
        return self._set_speed(12000000)

    def _set_speed(self, speed):
        power = min(4,max(0,math.floor(math.log2(speed/750000))))
        self.xpcu_set_jtag_speed(4-power)
        return 750000 * (2**power)

    @staticmethod
    def _count_tdo_bits(data, bit_count):
        return bin(sum([((ord(data[i*2+1:i*2+2])>>4) &
                         (( 1<< min(4, bit_count-(i*4)) )-1) )<<4*i
                        for i in range(int(len(data)/2))])).count('1')

    @staticmethod
    def _decode_tdo_bits(ret, *, bit_return_count=None, bit_count=None):
        if bit_return_count is None:
            bit_return_count = XilinxPC1Driver._count_tdo_bits(data,
                                                               bit_count)

        bytes_wanted = int(math.ceil(bit_return_count/8.0))
        bytes_expected = bytes_wanted +(1 if bytes_wanted%2 else 0)

        final_group_index = (bit_return_count-(bit_return_count%32))//8
        retiter = iter(ret[:final_group_index])
        fullgroups = [bytes(elem[::-1]) for elem in
                      zip(retiter, retiter, retiter, retiter)][::-1]
        other=ret[final_group_index:][::-1]
        other_bits = Bitarray()
        other_bits.frombytes(other)
        other_bits = other_bits[:bit_return_count-(8*final_group_index)]

        reordered_data = b"".join(fullgroups)
        raw_bits = Bitarray()
        raw_bits.frombytes(reordered_data)
        raw_bits = other_bits + raw_bits

        return raw_bits

__filter__ = [((0x03FD, 0x000D),XilinxPC1Driver),
              ((0x03FD, 0x0008),XilinxPC1Driver)]
<|endoftext|>