<commit_before>

import glob
import io
import json
import os
import sys
import traceback

from codecs import utf_8_encode
from functools import reduce
from optparse import OptionParser

from citeproc import CitationStylesStyle, CitationStylesBibliography
from citeproc.source import Citation, CitationItem, Locator
from citeproc.source.json import CiteProcJSON

TESTS_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                          os.path.pardir, os.path.pardir,
                                          'citeproc-test', 'processor-tests',
                                          'machines'))

class ProcessorTest(object):
    def __init__(self, filename):
        with open(filename, encoding='UTF-8') as f:
            self.json_data = json.load(f)

        csl_io = io.BytesIO(utf_8_encode(self.json_data['csl'])[0])
        self.style = CitationStylesStyle(csl_io)
        self.references = [item['id'] for item in self.json_data['input']]
        self.references_dict = CiteProcJSON(self.json_data['input'])
        self.bibliography = CitationStylesBibliography(self.style,
                                                       self.references_dict)
        self.expected = self.json_data['result'].splitlines()

    def execute(self):
        if self.json_data['citation_items']:
            citations = [self.parse_citation(item)
                         for item in self.json_data['citation_items']]
        elif self.json_data['citations']:
            citations = []
            for cit in self.json_data['citations']:
                cit = cit[0]
                citation_items = [self.parse_citation_item(cititem)
                                  for cititem in cit['citationItems']]
                citation = Citation(citation_items)
                citation.key = cit['citationID']
                citation.note_index = cit['properties']['noteIndex']
                citations.append(citation)
        elif self.json_data['bibentries']:
            citation_items = [self.parse_citation_item({'id': entry})
                              for entry in self.json_data['bibentries'][-1]]
            citations = [Citation(citation_items)]
        else:
            if self.json_data['mode'] == 'citation':
                self.references.sort()
            citation_items = [self.parse_citation_item({'id': ref})
                              for ref in self.references]
            citations = [Citation(citation_items)]

        for citation in citations:
            self.bibliography.register(citation)

        if self.style.has_bibliography():
            self.bibliography.sort()

        results = []
        do_nothing = lambda x: None     # callback passed to cite()
        if self.json_data['mode'] == 'citation':
            if self.json_data['citations']:
                for i, citation in enumerate(citations):
                    if i == len(citations) - 1:
                        dots_or_other = '>>'
                    else:
                        dots_or_other = '..'
                    results.append('{}[{}] '.format(dots_or_other, i) +
                                   self.bibliography.cite(citation, do_nothing))
            else:
                for citation in citations:
                    results.append(self.bibliography.cite(citation, do_nothing))
        elif self.json_data['mode'] in ('bibliography', 'bibliography-nosort'):
            results.append(self.bibliography.bibliography())

        return results

    def parse_citation(self, citation_data):
        citation_items = []
        for item in citation_data:
            citation_item = self.parse_citation_item(item)
            citation_items.append(citation_item)

        return Citation(citation_items)

    def parse_citation_item(self, citation_item_data):
        options = {}
        for key, value in citation_item_data.items():
            python_key = key.replace('-', '_')
            if python_key == 'id':
                reference_key = value
                continue
            elif python_key == 'locator':
                try:
                    options['locator'] = Locator(citation_item_data['label'],
                                                 value)
                except KeyError:
                    # some tests don't specify the label
                    options['locator'] = Locator(None, value)
            elif python_key == 'label':
                pass
            else:
                options[python_key] = value

        return CitationItem(reference_key, **options)

def main():
    usage = ('usage: %prog [options] glob_pattern\n\n'
             'glob_pattern limits the tests that are executed, for example:\n'
             '  %prog *Sort*\n'
             'runs only test fixtures that have "Sort" in the filename')
    parser = OptionParser(usage)
    parser.add_option('-m', '--max', dest='max', default=-1,
                      help='run maximally MAX tests', metavar='MAX')
    parser.add_option('-r', '--raise', dest='catch_exceptions', default=True,
                      action='store_false',
                      help='exceptions are not caught (aborts program)')
    parser.add_option('-f', '--file', dest='file', default=None,
                      help='write output to FILE', metavar='FILE')
    (options, args) = parser.parse_args()

    try:
        destination = open(options.file, 'w', encoding='utf-8')
        sys.stderr = destination
    except TypeError:
        destination = sys.stdout

    try:
        glob_pattern = args[0]
        filter_tests = False
    except IndexError:
        glob_pattern = '*'
        filter_tests = True

    total = {}
    passed = {}
    max_tests = int(options.max)

    count = 0
    for filename in glob.glob(os.path.join(TESTS_PATH,
                                           '{}.json'.format(glob_pattern))):
        category = os.path.basename(filename).split('_')[0]
        passed[category] = passed.get(category, 0)
        if count == max_tests:
            break

        try:
            t = ProcessorTest(filename)
            if filter_tests and (t.json_data['mode'] == 'bibliography-header' or
                                 t.json_data['bibsection']):
                continue
            total[category] = total.get(category, 0) + 1
            count += 1
            print('>>> Testing {}'.format(os.path.basename(filename)),
                  file=destination)
            print('EXP: ' + '\n     '.join(t.expected), file=destination)

            results = t.execute()
            results = reduce(lambda x, y: x+y,
                             [item.split('\n') for item in results])
            results = [item.replace('&', '&')
                       for item in results]
            print('RES: ' + '\n     '.join(results), file=destination)
            if results == t.expected:
                passed[category] += 1
                print('<<< SUCCESS\n', file=destination)
            else:
                print('<<< FAILED\n', file=destination)
            del t
        except Exception as e:
            print('Exception in', os.path.basename(filename), file=destination)
            if options.catch_exceptions:
                traceback.print_exc(file=destination)
            else:
                raise

    for category in sorted(total.keys()):
        print('{} passed {}/{}'.format(category, passed[category],
                                       total[category]), file=destination)

    print('total: {}/{}'.format(sum(passed.values()), sum(total.values())),
          file=destination)

if __name__ == '__main__':
    main()
<commit_msg>pretty print the test results<commit_after>

import glob
import io
import json
import os
import sys
import traceback

from codecs import utf_8_encode
from functools import reduce
from optparse import OptionParser

from citeproc import CitationStylesStyle, CitationStylesBibliography
from citeproc.source import Citation, CitationItem, Locator
from citeproc.source.json import CiteProcJSON

TESTS_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                          os.path.pardir, os.path.pardir,
                                          'citeproc-test', 'processor-tests',
                                          'machines'))

class ProcessorTest(object):
    def __init__(self, filename):
        with open(filename, encoding='UTF-8') as f:
            self.json_data = json.load(f)

        csl_io = io.BytesIO(utf_8_encode(self.json_data['csl'])[0])
        self.style = CitationStylesStyle(csl_io)
        self.references = [item['id'] for item in self.json_data['input']]
        self.references_dict = CiteProcJSON(self.json_data['input'])
        self.bibliography = CitationStylesBibliography(self.style,
                                                       self.references_dict)
        self.expected = self.json_data['result'].splitlines()

    def execute(self):
        if self.json_data['citation_items']:
            citations = [self.parse_citation(item)
                         for item in self.json_data['citation_items']]
        elif self.json_data['citations']:
            citations = []
            for cit in self.json_data['citations']:
                cit = cit[0]
                citation_items = [self.parse_citation_item(cititem)
                                  for cititem in cit['citationItems']]
                citation = Citation(citation_items)
                citation.key = cit['citationID']
                citation.note_index = cit['properties']['noteIndex']
                citations.append(citation)
        elif self.json_data['bibentries']:
            citation_items = [self.parse_citation_item({'id': entry})
                              for entry in self.json_data['bibentries'][-1]]
            citations = [Citation(citation_items)]
        else:
            if self.json_data['mode'] == 'citation':
                self.references.sort()
            citation_items = [self.parse_citation_item({'id': ref})
                              for ref in self.references]
            citations = [Citation(citation_items)]

        for citation in citations:
            self.bibliography.register(citation)

        if self.style.has_bibliography():
            self.bibliography.sort()

        results = []
        do_nothing = lambda x: None     # callback passed to cite()
        if self.json_data['mode'] == 'citation':
            if self.json_data['citations']:
                for i, citation in enumerate(citations):
                    if i == len(citations) - 1:
                        dots_or_other = '>>'
                    else:
                        dots_or_other = '..'
                    results.append('{}[{}] '.format(dots_or_other, i) +
                                   self.bibliography.cite(citation, do_nothing))
            else:
                for citation in citations:
                    results.append(self.bibliography.cite(citation, do_nothing))
        elif self.json_data['mode'] in ('bibliography', 'bibliography-nosort'):
            results.append(self.bibliography.bibliography())

        return results

    def parse_citation(self, citation_data):
        citation_items = []
        for item in citation_data:
            citation_item = self.parse_citation_item(item)
            citation_items.append(citation_item)

        return Citation(citation_items)

    def parse_citation_item(self, citation_item_data):
        options = {}
        for key, value in citation_item_data.items():
            python_key = key.replace('-', '_')
            if python_key == 'id':
                reference_key = value
                continue
            elif python_key == 'locator':
                try:
                    options['locator'] = Locator(citation_item_data['label'],
                                                 value)
                except KeyError:
                    # some tests don't specify the label
                    options['locator'] = Locator(None, value)
            elif python_key == 'label':
                pass
            else:
                options[python_key] = value

        return CitationItem(reference_key, **options)

def main():
    usage = ('usage: %prog [options] glob_pattern\n\n'
             'glob_pattern limits the tests that are executed, for example:\n'
             '  %prog *Sort*\n'
             'runs only test fixtures that have "Sort" in the filename')
    parser = OptionParser(usage)
    parser.add_option('-m', '--max', dest='max', default=-1,
                      help='run maximally MAX tests', metavar='MAX')
    parser.add_option('-r', '--raise', dest='catch_exceptions', default=True,
                      action='store_false',
                      help='exceptions are not caught (aborts program)')
    parser.add_option('-f', '--file', dest='file', default=None,
                      help='write output to FILE', metavar='FILE')
    (options, args) = parser.parse_args()

    try:
        destination = open(options.file, 'w', encoding='utf-8')
        sys.stderr = destination
    except TypeError:
        destination = sys.stdout

    try:
        glob_pattern = args[0]
        filter_tests = False
    except IndexError:
        glob_pattern = '*'
        filter_tests = True

    total = {}
    passed = {}
    max_tests = int(options.max)

    count = 0
    for filename in glob.glob(os.path.join(TESTS_PATH,
                                           '{}.json'.format(glob_pattern))):
        category = os.path.basename(filename).split('_')[0]
        passed[category] = passed.get(category, 0)
        if count == max_tests:
            break

        try:
            t = ProcessorTest(filename)
            if filter_tests and (t.json_data['mode'] == 'bibliography-header' or
                                 t.json_data['bibsection']):
                continue
            total[category] = total.get(category, 0) + 1
            count += 1
            print('>>> Testing {}'.format(os.path.basename(filename)),
                  file=destination)
            print('EXP: ' + '\n     '.join(t.expected), file=destination)

            results = t.execute()
            results = reduce(lambda x, y: x+y,
                             [item.split('\n') for item in results])
            results = [item.replace('&', '&')
                       for item in results]
            print('RES: ' + '\n     '.join(results), file=destination)
            if results == t.expected:
                passed[category] += 1
                print('<<< SUCCESS\n', file=destination)
            else:
                print('<<< FAILED\n', file=destination)
            del t
        except Exception as e:
            print('Exception in', os.path.basename(filename), file=destination)
            if options.catch_exceptions:
                traceback.print_exc(file=destination)
            else:
                raise

    def print_result(name, passed, total):
        print('{:<13} {:>3} / {:>3} ({:>4.0%})'
              .format(name, passed, total, passed / total), file=destination)

    for category in sorted(total.keys()):
        print_result(category, passed[category], total[category])
    print_result('total', sum(passed.values()), sum(total.values()))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#   Copyright 2020 Google LLC

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Define version number here and read it from setup.py automatically"""
__version__ = "0.0.0.dev1"
<commit_msg>Release version<commit_after>#   Copyright 2020 Google LLC

#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Define version number here and read it from setup.py automatically"""
__version__ = "0.1.0"
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import json
import sys
import os

(_, fn) = sys.argv

## Coq can produce very deep JSON data structures..
sys.setrecursionlimit(10000)

## XXX hack for now
import_prefix = 'codegen/'

this_pkgname = None

with open(fn) as f:
  s = f.read()
  d = json.loads(s)

def coqname_in_mod(s):
  return 'Coq_' + s.replace("'", "_prime").replace(".", "_")

def coqname(s):
  global this_pkgname
  if '.' in s:
    (mod, rest) = s.split('.', 1)
    if mod == this_pkgname:
      return coqname_in_mod(rest)
    else:
      return '%s.%s' % (mod, coqname_in_mod(rest))
  else:
    return coqname_in_mod(s)

varname_ctr = 0
def varname():
  global varname_ctr
  varname_ctr += 1
  return "__v_%d" % varname_ctr

def gen_fix_expr(names, lambdas):
  s = ''
  s += 'func (__fixarg CoqT) CoqT {\n'

  for name in names:
    s += '  var %s CoqT\n' % coqname(name)

  for (name, lambd) in zip(names, lambdas):
    s += gen_expr_assign(lambd, coqname(name))

  s += '  return %s.(func (CoqT) CoqT)(__fixarg)\n' % coqname(names[-1])
  s += '}\n'
  return s

def gen_expr_assign(e, retname):
  s = ''

  if e['what'] == 'expr:lambda':
    lambda_name = retname

    closure_vars = []
    for argname in e['argnames']:
      closure_var = varname()
      closure_vars = [closure_var] + closure_vars
      s += '%s = func (%s CoqT) CoqT {\n' % (lambda_name, coqname(argname))
      s += '  var %s CoqT\n' % closure_var
      lambda_name = closure_var

    s += gen_expr_assign(e['body'], lambda_name)

    for closure_var in closure_vars:
      s += '  return %s\n' % closure_var
      s += '}\n'

  elif e['what'] == 'expr:fix':
    fixnames = [x['name'] for x in e['funcs']]
    fixlambdas = [x['body'] for x in e['funcs']]
    s += '%s = %s\n' % (retname, gen_fix_expr(fixnames, fixlambdas))

  elif e['what'] == 'expr:case':
    switchvar = varname()
    s += 'var %s CoqT\n' % switchvar
    s += gen_expr_assign(e['expr'], switchvar)
    s += 'switch __typesw := (%s).(type) {\n' % switchvar

    have_default = False
    for case in e['cases']:
      body = gen_expr_assign(case['body'], retname)
      pat = case['pat']

      if pat['what'] == 'pat:constructor':
        s += 'case *%s:\n' % coqname(pat['name'])
        for idx, argname in enumerate(pat['argnames']):
          s += 'var %s CoqT = __typesw.A%d\n' % (coqname(argname), idx)
          s += 'var _ = %s\n' % coqname(argname)
        s += body

      elif pat['what'] == 'pat:wild':
        s += 'default:\n'
        s += '  var _ = __typesw\n'
        s += body
        have_default = True

      elif pat['what'] == 'pat:rel':
        s += 'default:\n'
        s += '  var _ = __typesw\n'
        s += '  var %s CoqT\n' % coqname(pat['name'])
        s += '  %s = %s\n' % (coqname(pat['name']), switchvar)
        s += body
        have_default = True

      else:
        s += 'UNKNOWN PAT %s' % pat['what']

    if not have_default:
      s += 'default:\n'
      s += '  var _ = __typesw\n'
      s += '  %s = nil\n' % retname
      s += '  panic("switch returned")\n'
    s += '}\n'

  elif e['what'] == 'expr:rel':
    s += '%s = %s\n' % (retname, coqname(e['name']))

  elif e['what'] == 'expr:global':
    s += '%s = %s\n' % (retname, coqname(e['name']))

  elif e['what'] == 'expr:constructor':
    arg_vars = []
    for a in e['args']:
      argvar = varname()
      arg_vars.append(argvar)
      s += 'var %s CoqT\n' % argvar
      s += gen_expr_assign(a, argvar)

    s += '%s = &%s{ %s }\n' % (retname, coqname(e['name']), ', '.join(arg_vars))

  elif e['what'] == 'expr:exception':
    s += '%s = nil\n' % retname
    s += 'panic("%s")\n' % e['msg']

  elif e['what'] == 'expr:apply':
    funvar = varname()
    s += 'var %s CoqT\n' % funvar
    s += gen_expr_assign(e['func'], funvar)

    arg_vars = []
    for a in e['args']:
      argvar = varname()
      arg_vars.append(argvar)
      s += 'var %s CoqT\n' % argvar
      s += gen_expr_assign(a, argvar)

    apply_expr = funvar
    for arg in arg_vars:
      apply_expr = '(%s).(func (CoqT) CoqT)(%s)' % (apply_expr, arg)

    s += '%s = %s\n' % (retname, apply_expr)

  elif e['what'] == 'expr:dummy':
    s += '%s = CoqDummy\n' % retname

  elif e['what'] == 'expr:let':
    s += 'var %s CoqT\n' % coqname(e['name'])
    s += gen_expr_assign(e['nameval'], coqname(e['name']))
    s += gen_expr_assign(e['body'], retname)

  elif e['what'] == 'expr:axiom':
    s += '%s = nil\n' % retname
    s += 'panic("Axiom not realized")\n'

  elif e['what'] == 'expr:coerce':
    s += gen_expr_assign(e['value'], retname)

  else:
    s += 'UNKNOWN EXPR %s\n' % e['what']

  return s

def gen_header(d):
  global this_pkgname
  this_pkgname = d['name']

  s = ''
  s += 'package %s\n' % d['name']
  s += 'import . "gocoq"\n'
  for modname in d['used_modules']:
    s += 'import "%s%s"\n' % (import_prefix, modname)
  return s

def gen_ind(dec):
  s = ''
  for c in dec['constructors']:
    s += 'type %s struct {\n' % coqname(c['name'])
    for idx, typ in enumerate(c['argtypes']):
      s += '  A%d CoqT\n' % idx
    s += '}\n'
  return s

def gen_term(dec):
  v = varname()
  s = ''
  s += 'func () CoqT {\n'
  s += '  var %s CoqT\n' % v
  s += gen_expr_assign(dec['value'], v)
  s += '  return %s\n' % v
  s += '} ()'

  return 'var %s CoqT = %s\n' % (coqname(dec['name']), s)

def gen_fix(dec):
  e = gen_fix_expr([dec['name']], [dec['value']])
  return 'var %s CoqT = %s\n' % (coqname(dec['name']), e)

print(gen_header(d))

for dec in d['declarations']:
  if dec['what'] == 'decl:type':
    pass
  elif dec['what'] == 'decl:term':
    print(gen_term(dec))
  elif dec['what'] == 'decl:fix':
    print(gen_fix(dec))
  elif dec['what'] == 'decl:ind':
    print(gen_ind(dec))
  else:
    assert False, dec
<commit_msg>hack to get around unused import statements in Go<commit_after>#!/usr/bin/env python3

import json
import sys
import os

(_, fn) = sys.argv

## Coq can produce very deep JSON data structures..
sys.setrecursionlimit(10000)

## XXX hack for now
import_prefix = 'codegen/'

this_pkgname = None

with open(fn) as f:
  s = f.read()
  d = json.loads(s)

def coqname_in_mod(s):
  return 'Coq_' + s.replace("'", "_prime").replace(".", "_")

def coqname(s):
  global this_pkgname
  if '.' in s:
    (mod, rest) = s.split('.', 1)
    if mod == this_pkgname:
      return coqname_in_mod(rest)
    else:
      return '%s.%s' % (mod, coqname_in_mod(rest))
  else:
    return coqname_in_mod(s)

varname_ctr = 0
def varname():
  global varname_ctr
  varname_ctr += 1
  return "__v_%d" % varname_ctr

def gen_fix_expr(names, lambdas):
  s = ''
  s += 'func (__fixarg CoqT) CoqT {\n'

  for name in names:
    s += '  var %s CoqT\n' % coqname(name)

  for (name, lambd) in zip(names, lambdas):
    s += gen_expr_assign(lambd, coqname(name))

  s += '  return %s.(func (CoqT) CoqT)(__fixarg)\n' % coqname(names[-1])
  s += '}\n'
  return s

def gen_expr_assign(e, retname):
  s = ''

  if e['what'] == 'expr:lambda':
    lambda_name = retname

    closure_vars = []
    for argname in e['argnames']:
      closure_var = varname()
      closure_vars = [closure_var] + closure_vars
      s += '%s = func (%s CoqT) CoqT {\n' % (lambda_name, coqname(argname))
      s += '  var %s CoqT\n' % closure_var
      lambda_name = closure_var

    s += gen_expr_assign(e['body'], lambda_name)

    for closure_var in closure_vars:
      s += '  return %s\n' % closure_var
      s += '}\n'

  elif e['what'] == 'expr:fix':
    fixnames = [x['name'] for x in e['funcs']]
    fixlambdas = [x['body'] for x in e['funcs']]
    s += '%s = %s\n' % (retname, gen_fix_expr(fixnames, fixlambdas))

  elif e['what'] == 'expr:case':
    switchvar = varname()
    s += 'var %s CoqT\n' % switchvar
    s += gen_expr_assign(e['expr'], switchvar)
    s += 'switch __typesw := (%s).(type) {\n' % switchvar

    have_default = False
    for case in e['cases']:
      body = gen_expr_assign(case['body'], retname)
      pat = case['pat']

      if pat['what'] == 'pat:constructor':
        s += 'case *%s:\n' % coqname(pat['name'])
        for idx, argname in enumerate(pat['argnames']):
          s += 'var %s CoqT = __typesw.A%d\n' % (coqname(argname), idx)
          s += 'var _ = %s\n' % coqname(argname)
        s += body

      elif pat['what'] == 'pat:wild':
        s += 'default:\n'
        s += '  var _ = __typesw\n'
        s += body
        have_default = True

      elif pat['what'] == 'pat:rel':
        s += 'default:\n'
        s += '  var _ = __typesw\n'
        s += '  var %s CoqT\n' % coqname(pat['name'])
        s += '  %s = %s\n' % (coqname(pat['name']), switchvar)
        s += body
        have_default = True

      else:
        s += 'UNKNOWN PAT %s' % pat['what']

    if not have_default:
      s += 'default:\n'
      s += '  var _ = __typesw\n'
      s += '  %s = nil\n' % retname
      s += '  panic("switch returned")\n'
    s += '}\n'

  elif e['what'] == 'expr:rel':
    s += '%s = %s\n' % (retname, coqname(e['name']))

  elif e['what'] == 'expr:global':
    s += '%s = %s\n' % (retname, coqname(e['name']))

  elif e['what'] == 'expr:constructor':
    arg_vars = []
    for a in e['args']:
      argvar = varname()
      arg_vars.append(argvar)
      s += 'var %s CoqT\n' % argvar
      s += gen_expr_assign(a, argvar)

    s += '%s = &%s{ %s }\n' % (retname, coqname(e['name']), ', '.join(arg_vars))

  elif e['what'] == 'expr:exception':
    s += '%s = nil\n' % retname
    s += 'panic("%s")\n' % e['msg']

  elif e['what'] == 'expr:apply':
    funvar = varname()
    s += 'var %s CoqT\n' % funvar
    s += gen_expr_assign(e['func'], funvar)

    arg_vars = []
    for a in e['args']:
      argvar = varname()
      arg_vars.append(argvar)
      s += 'var %s CoqT\n' % argvar
      s += gen_expr_assign(a, argvar)

    apply_expr = funvar
    for arg in arg_vars:
      apply_expr = '(%s).(func (CoqT) CoqT)(%s)' % (apply_expr, arg)

    s += '%s = %s\n' % (retname, apply_expr)

  elif e['what'] == 'expr:dummy':
    s += '%s = CoqDummy\n' % retname

  elif e['what'] == 'expr:let':
    s += 'var %s CoqT\n' % coqname(e['name'])
    s += gen_expr_assign(e['nameval'], coqname(e['name']))
    s += gen_expr_assign(e['body'], retname)

  elif e['what'] == 'expr:axiom':
    s += '%s = nil\n' % retname
    s += 'panic("Axiom not realized")\n'

  elif e['what'] == 'expr:coerce':
    s += gen_expr_assign(e['value'], retname)

  else:
    s += 'UNKNOWN EXPR %s\n' % e['what']

  return s

def gen_header(d):
  global this_pkgname
  this_pkgname = d['name']

  s = ''
  s += 'package %s\n' % d['name']
  s += 'import . "gocoq"\n'
  for modname in d['used_modules']:
    s += 'import "%s%s"\n' % (import_prefix, modname)
  s += 'var Coq2go_unused bool = true'
  for modname in d['used_modules']:
    s += ' && %s.Coq2go_unused' % modname
  s += '\n'
  return s

def gen_ind(dec):
  s = ''
  for c in dec['constructors']:
    s += 'type %s struct {\n' % coqname(c['name'])
    for idx, typ in enumerate(c['argtypes']):
      s += '  A%d CoqT\n' % idx
    s += '}\n'
  return s

def gen_term(dec):
  v = varname()
  s = ''
  s += 'func () CoqT {\n'
  s += '  var %s CoqT\n' % v
  s += gen_expr_assign(dec['value'], v)
  s += '  return %s\n' % v
  s += '} ()'

  return 'var %s CoqT = %s\n' % (coqname(dec['name']), s)

def gen_fix(dec):
  e = gen_fix_expr([dec['name']], [dec['value']])
  return 'var %s CoqT = %s\n' % (coqname(dec['name']), e)

print(gen_header(d))

for dec in d['declarations']:
  if dec['what'] == 'decl:type':
    pass
  elif dec['what'] == 'decl:term':
    print(gen_term(dec))
  elif dec['what'] == 'decl:fix':
    print(gen_fix(dec))
  elif dec['what'] == 'decl:ind':
    print(gen_ind(dec))
  else:
    assert False, dec
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    flaskext.testing
    ~~~~~~~~~~~~~~~~~

    Flask unittest integration.

    :copyright: (c) 2010 by Dan Jacob.
    :license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, with_statement

import StringIO
import unittest
import twill

from werkzeug import cached_property

# Use Flask's preferred JSON module so that our runtime behavior matches.
from flask import json_available
if json_available:
    from flask import json

try:
    # we'll use signals for template-related tests if
    # available in this version of Flask
    import blinker
    from flask import template_rendered
    _is_signals = True
except ImportError:
    _is_signals = False

__all__ = ["TestCase", "TwillMixin"]

class ContextVariableDoesNotExist(Exception):
    pass

class JsonResponseMixin(object):
    """
    Mixin with testing helper methods
    """
    @cached_property
    def json(self):
        if not json_available:
            raise NotImplementedError
        return json.loads(self.data)

def _make_test_response(response_class):
    class TestResponse(response_class, JsonResponseMixin):
        pass

    return TestResponse

class TestCase(unittest.TestCase):

    def create_app(self):
        """
        Create your Flask app here, with any
        configuration you need.
        """
        raise NotImplementedError

    def __call__(self, result=None):
        """
        Does the required setup, doing it here
        means you don't have to call super.setUp
        in subclasses.
        """
        try:
            self._pre_setup()
            super(TestCase, self).__call__(result)
        finally:
            self._post_teardown()

    def _pre_setup(self):
        self.app = self._ctx = None

        self.app = self.create_app()

        self._orig_response_class = self.app.response_class
        self.app.response_class = _make_test_response(self.app.response_class)

        self.client = self.app.test_client()

        self._ctx = self.app.test_request_context()
        self._ctx.push()

        self.templates = []
        if _is_signals:
            template_rendered.connect(self._add_template)

    def _add_template(self, app, template, context):
        self.templates.append((template, context))

    def _post_teardown(self):
        if self._ctx is not None:
            self._ctx.pop()
        if self.app is not None:
            self.app.response_class = self._orig_response_class
        if _is_signals:
            template_rendered.disconnect(self._add_template)

    def assertTemplateUsed(self, name):
        """
        Checks if a given template is used in the request.
        Only works if your version of Flask has signals
        support (0.6+) and blinker is installed.

        :versionadded: 0.2
        :param name: template name
        """
        if not _is_signals:
            raise RuntimeError("Signals not supported")

        for template, context in self.templates:
            if template.name == name:
                return True
        raise AssertionError("template %s not used" % name)

    assert_template_used = assertTemplateUsed

    def get_context_variable(self, name):
        """
        Returns a variable from the context passed to the
        template. Only works if your version of Flask
        has signals support (0.6+) and blinker is installed.

        Raises a ContextVariableDoesNotExist exception if does
        not exist in context.

        :versionadded: 0.2
        :param name: name of variable
        """
        if not _is_signals:
            raise RuntimeError("Signals not supported")

        for template, context in self.templates:
            if name in context:
                return context[name]
        raise ContextVariableDoesNotExist

    def assertContext(self, name, value):
        """
        Checks if given name exists in the template context
        and equals the given value.

        :versionadded: 0.2
        :param name: name of context variable
        :param value: value to check against
        """

        try:
            self.assertEqual(self.get_context_variable(name), value)
        except ContextVariableDoesNotExist:
            self.fail("Context variable does not exist: %s" % name)

    assert_context = assertContext

    def assertRedirects(self, response, location):
        """
        Checks if response is an HTTP redirect to the
        given location.

        :param response: Flask response
        :param location: relative URL (i.e. without **http://localhost**)
        """
        self.assertTrue(response.status_code in (301, 302))
        self.assertEqual(response.location, "http://localhost" + location)

    assert_redirects = assertRedirects

    def assertStatus(self, response, status_code):
        """
        Helper method to check matching response status.

        :param response: Flask response
        :param status_code: response status code (e.g. 200)
        """
        self.assertEqual(response.status_code, status_code)

    assert_status = assertStatus

    def assert200(self, response):
        """
        Checks if response status code is 200

        :param response: Flask response
        """

        self.assertStatus(response, 200)

    assert_200 = assert200

    def assert400(self, response):
        """
        Checks if response status code is 400

        :versionadded: 0.2.5
        :param response: Flask response
        """

        self.assertStatus(response, 400)

    assert_400 = assert400

    def assert401(self, response):
        """
        Checks if response status code is 401

        :versionadded: 0.2.1
        :param response: Flask response
        """

        self.assertStatus(response, 401)

    assert_401 = assert401

    def assert403(self, response):
        """
        Checks if response status code is 403

        :versionadded: 0.2
        :param response: Flask response
        """

        self.assertStatus(response, 403)

    assert_403 = assert403

    def assert404(self, response):
        """
        Checks if response status code is 404

        :param response: Flask response
        """

        self.assertStatus(response, 404)

    assert_404 = assert404

    def assert405(self, response):
        """
        Checks if response status code is 405

        :versionadded: 0.2
        :param response: Flask response
        """

        self.assertStatus(response, 405)

    assert_405 = assert405

class Twill(object):
    """

    :versionadded: 0.3

    Twill wrapper utility class.

    Creates a Twill ``browser`` instance and handles
    WSGI intercept.

    Usage::

        t = Twill(self.app)
        with t:
            t.browser.go("/")
            t.url("/")

    """
    def __init__(self, app, host='127.0.0.1', port=5000, scheme='http'):
        self.app = app
        self.host = host
        self.port = port
        self.scheme = scheme

        self.browser = twill.get_browser()

    def __enter__(self):
        twill.set_output(StringIO.StringIO())
        twill.commands.clear_cookies()
        twill.add_wsgi_intercept(self.host,
                                 self.port,
                                 lambda: self.app)

        return self

    def __exit__(self, exc_type, exc_value, tb):
        twill.remove_wsgi_intercept(self.host,
                                    self.port)

        twill.commands.reset_output()

    def url(self, url):
        """
        Makes complete URL based on host, port and scheme
        Twill settings.

        :param url: relative URL
        """
        return "%s://%s:%d%s" % (self.scheme,
                                 self.host,
                                 self.port,
                                 url)

class TwillTestCase(TestCase):
    """
    :deprecated: use Twill helper class instead.

    Creates a Twill ``browser`` instance and handles
    WSGI intercept.
    """

    twill_host = "127.0.0.1"
    twill_port = 5000
    twill_scheme = "http"

    def _pre_setup(self):
        super(TwillTestCase, self)._pre_setup()
        twill.set_output(StringIO.StringIO())
        twill.commands.clear_cookies()
        twill.add_wsgi_intercept(self.twill_host,
                                 self.twill_port,
                                 lambda: self.app)

        self.browser = twill.get_browser()

    def _post_teardown(self):

        twill.remove_wsgi_intercept(self.twill_host,
                                    self.twill_port)

        twill.commands.reset_output()

        super(TwillTestCase, self)._post_teardown()

    def make_twill_url(self, url):
        """
        Makes complete URL based on host, port and scheme
        Twill settings.

        :param url: relative URL
        """
        return "%s://%s:%d%s" % (self.twill_scheme,
                                 self.twill_host,
                                 self.twill_port,
                                 url)
<commit_msg>Add some exclude for code coverage<commit_after># -*- coding: utf-8 -*-
"""
    flaskext.testing
    ~~~~~~~~~~~~~~~~~

    Flask unittest integration.

    :copyright: (c) 2010 by Dan Jacob.
    :license: BSD, see LICENSE for more details.
"""
from __future__ import absolute_import, with_statement

import StringIO
import unittest
import twill

from werkzeug import cached_property

# Use Flask's preferred JSON module so that our runtime behavior matches.
from flask import json_available
if json_available:
    from flask import json

# we'll use signals for template-related tests if
# available in this version of Flask
try:
    import blinker
    from flask import template_rendered
    _is_signals = True
except ImportError: # pragma: no cover
    _is_signals = False

__all__ = ["TestCase", "TwillMixin"]

class ContextVariableDoesNotExist(Exception):
    pass

class JsonResponseMixin(object):
    """
    Mixin with testing helper methods
    """
    @cached_property
    def json(self):
        if not json_available: # pragma: no cover
            raise NotImplementedError
        return json.loads(self.data)

def _make_test_response(response_class):
    class TestResponse(response_class, JsonResponseMixin):
        pass

    return TestResponse

class TestCase(unittest.TestCase):

    def create_app(self):
        """
        Create your Flask app here, with any
        configuration you need.
        """
        raise NotImplementedError

    def __call__(self, result=None):
        """
        Does the required setup, doing it here
        means you don't have to call super.setUp
        in subclasses.
        """
        try:
            self._pre_setup()
            super(TestCase, self).__call__(result)
        finally:
            self._post_teardown()

    def _pre_setup(self):
        self.app = self._ctx = None

        self.app = self.create_app()

        self._orig_response_class = self.app.response_class
        self.app.response_class = _make_test_response(self.app.response_class)

        self.client = self.app.test_client()

        self._ctx = self.app.test_request_context()
        self._ctx.push()

        self.templates = []
        if _is_signals:
            template_rendered.connect(self._add_template)

    def _add_template(self, app, template, context):
        self.templates.append((template, context))

    def _post_teardown(self):
        if self._ctx is not None:
            self._ctx.pop()
        if self.app is not None:
            self.app.response_class = self._orig_response_class
        if _is_signals:
            template_rendered.disconnect(self._add_template)

    def assertTemplateUsed(self, name):
        """
        Checks if a given template is used in the request.
        Only works if your version of Flask has signals
        support (0.6+) and blinker is installed.

        :versionadded: 0.2
        :param name: template name
        """
        if not _is_signals:
            raise RuntimeError("Signals not supported")

        for template, context in self.templates:
            if template.name == name:
                return True
        raise AssertionError("template %s not used" % name)

    assert_template_used = assertTemplateUsed

    def get_context_variable(self, name):
        """
        Returns a variable from the context passed to the
        template. Only works if your version of Flask
        has signals support (0.6+) and blinker is installed.

        Raises a ContextVariableDoesNotExist exception if does
        not exist in context.

        :versionadded: 0.2
        :param name: name of variable
        """
        if not _is_signals:
            raise RuntimeError("Signals not supported")

        for template, context in self.templates:
            if name in context:
                return context[name]
        raise ContextVariableDoesNotExist

    def assertContext(self, name, value):
        """
        Checks if given name exists in the template context
        and equals the given value.

        :versionadded: 0.2
        :param name: name of context variable
        :param value: value to check against
        """

        try:
            self.assertEqual(self.get_context_variable(name), value)
        except ContextVariableDoesNotExist:
            self.fail("Context variable does not exist: %s" % name)

    assert_context = assertContext

    def assertRedirects(self, response, location):
        """
        Checks if response is an HTTP redirect to the
        given location.

        :param response: Flask response
        :param location: relative URL (i.e. without **http://localhost**)
        """
        self.assertTrue(response.status_code in (301, 302))
        self.assertEqual(response.location, "http://localhost" + location)

    assert_redirects = assertRedirects

    def assertStatus(self, response, status_code):
        """
        Helper method to check matching response status.

        :param response: Flask response
        :param status_code: response status code (e.g. 200)
        """
        self.assertEqual(response.status_code, status_code)

    assert_status = assertStatus

    def assert200(self, response):
        """
        Checks if response status code is 200

        :param response: Flask response
        """

        self.assertStatus(response, 200)

    assert_200 = assert200

    def assert400(self, response):
        """
        Checks if response status code is 400

        :versionadded: 0.2.5
        :param response: Flask response
        """

        self.assertStatus(response, 400)

    assert_400 = assert400

    def assert401(self, response):
        """
        Checks if response status code is 401

        :versionadded: 0.2.1
        :param response: Flask response
        """

        self.assertStatus(response, 401)

    assert_401 = assert401

    def assert403(self, response):
        """
        Checks if response status code is 403

        :versionadded: 0.2
        :param response: Flask response
        """

        self.assertStatus(response, 403)

    assert_403 = assert403

    def assert404(self, response):
        """
        Checks if response status code is 404

        :param response: Flask response
        """

        self.assertStatus(response, 404)

    assert_404 = assert404

    def assert405(self, response):
        """
        Checks if response status code is 405

        :versionadded: 0.2
        :param response: Flask response
        """

        self.assertStatus(response, 405)

    assert_405 = assert405

class Twill(object):
    """

    :versionadded: 0.3

    Twill wrapper utility class.

    Creates a Twill ``browser`` instance and handles
    WSGI intercept.

    Usage::

        t = Twill(self.app)
        with t:
            t.browser.go("/")
            t.url("/")

    """
    def __init__(self, app, host='127.0.0.1', port=5000, scheme='http'):
        self.app = app
        self.host = host
        self.port = port
        self.scheme = scheme

        self.browser = twill.get_browser()

    def __enter__(self):
        twill.set_output(StringIO.StringIO())
        twill.commands.clear_cookies()
        twill.add_wsgi_intercept(self.host,
                                 self.port,
                                 lambda: self.app)

        return self

    def __exit__(self, exc_type, exc_value, tb):
        twill.remove_wsgi_intercept(self.host,
                                    self.port)

        twill.commands.reset_output()

    def url(self, url):
        """
        Makes complete URL based on host, port and scheme
        Twill settings.

        :param url: relative URL
        """
        return "%s://%s:%d%s" % (self.scheme,
                                 self.host,
                                 self.port,
                                 url)

class TwillTestCase(TestCase):
    """
    :deprecated: use Twill helper class instead.

    Creates a Twill ``browser`` instance and handles
    WSGI intercept.
    """

    twill_host = "127.0.0.1"
    twill_port = 5000
    twill_scheme = "http"

    def _pre_setup(self):
        super(TwillTestCase, self)._pre_setup()
        twill.set_output(StringIO.StringIO())
        twill.commands.clear_cookies()
        twill.add_wsgi_intercept(self.twill_host,
                                 self.twill_port,
                                 lambda: self.app)

        self.browser = twill.get_browser()

    def _post_teardown(self):

        twill.remove_wsgi_intercept(self.twill_host,
                                    self.twill_port)

        twill.commands.reset_output()

        super(TwillTestCase, self)._post_teardown()

    def make_twill_url(self, url):
        """
        Makes complete URL based on host, port and scheme
        Twill settings.

        :param url: relative URL
        """
        return "%s://%s:%d%s" % (self.twill_scheme,
                                 self.twill_host,
                                 self.twill_port,
                                 url)
<|endoftext|>


<commit_before>import csv

def main(vgi_repository='t51m', points=True):
    filter_out = {}  # dict for fast look-up
    with open("location_field/state_table.csv", "r") as fin:
        csvreader = csv.reader(fin)
        next(csvreader)
        for line in csvreader:
            if line[1] != 'Washington DC':
                for direction in ['', 'southern ', 'eastern ', 'northern ', 'central ', 'western ']:
                    filter_out[direction + line[1].lower()] = True  # e.g. Alabama
                    filter_out[direction + line[2].lower()] = True  # e.g. AL
                    filter_out[direction + line[1].lower() + ", usa"] = True
                    filter_out[direction + line[2].lower() + ", usa"] = True
                    filter_out[direction + line[1].lower() + ", us"] = True
                    filter_out[direction + line[2].lower() + ", us"] = True

    filter_out['america'] = True
    filter_out['etats-unis'] = True
    filter_out['usa'] = True
    filter_out['u.s.a.'] = True
    filter_out['us'] = True
    filter_out['u.s.'] = True
    filter_out['united states'] = True
    filter_out['united states of america'] = True
    filter_out['estados unidos'] = True

    filter_out['pacific northwest'] = True
    filter_out['the mitten'] = True
    filter_out['tejas'] = True
    filter_out['new england'] = True
    filter_out['lone star state'] = True

    filter_out['earth'] = True
    filter_out['nowhere'] = True
    filter_out['arg'] = True
    filter_out['central city'] = True
    filter_out['location'] = True
    filter_out['disney'] = True
    filter_out['clouds'] = True
    filter_out['area 51'] = True
    filter_out['westside'] = True
    filter_out['lol'] = True
    filter_out['house'] = True
    filter_out['krypton'] = True
    filter_out['pandora'] = True
    filter_out['cosmos'] = True
    filter_out['beach'] = True
    filter_out['happy'] = True
    filter_out['mars'] = True
    filter_out['bed'] = True
    filter_out['wonderland'] = True
    filter_out['south'] = True
    filter_out['nirvana'] = True
    filter_out['bdg'] = True
    filter_out['life'] = True
    filter_out['heart'] = True
    filter_out['indian'] = True
    filter_out['eastern'] = True
    filter_out['mlk'] = True
    filter_out['hope'] = True

    filter_out['badlands'] = True
    filter_out['dixie'] = True
    filter_out['san andreas'] = True
    filter_out['transylvania'] = True
    filter_out['belgique'] = True
    filter_out['pateros'] = True  # Manila
    filter_out['corsica'] = True
    filter_out['wimbledon'] = True
    filter_out['fsu'] = True
    filter_out['scandinavia'] = True
    filter_out['mhs'] = True
    filter_out['queen city'] = True # likely Cincinnati but a general term too...
    filter_out['ayrshire'] = True
    filter_out['alberta'] = True
    filter_out['newfoundland'] = True
    filter_out['bromley'] = True  # district in London

    with open('location_field/country_codes.tsv', 'r') as fin:
        csvreader = csv.reader(fin, delimiter='\t')
        header = next(csvreader)
        country_idx = header.index('Country')
        for line in csvreader:
            country = line[country_idx].lower().strip()
            filter_out[country] = True

    fix = {}
    if points:
        fix['washington dc'] = '(38.89511, -77.03637)'
        fix['twin cities, mn'] = '(44.96219, -93.178555)'  # Average of Minneaplis center and St. Paul center
        fix['twin cities, minnesota'] = '(44.96219, -93.178555)'
        fix['city of angels'] = '(34.05223, -118.24368)'
        fix['the city of angels'] = '(34.05223, -118.24368)'
        fix['phil'] = "(39.95233, -75.16379)"
        fix['delco'] = "(39.9168, -75.3989)"
        fix['steel city'] = "(40.44062, -79.99589)"
        fix['queens'] = "(40.76538, -73.81736)"
        for nyc_variant in ['nyc', 'new york, new york','new york city', 'new york, ny', 'ny, ny', 'the big apple']:
            fix[nyc_variant] = '(40.78343, -73.96625)'

    else:
        fix['washington dc'] = '11001'
        fix['twin cities, mn'] = '27053;27123'
        fix['twin cities, minnesota'] = '27053;27123'
        fix['city of angels'] = '06037'
        fix['the city of angels'] = '06037'
        fix['phil'] = '42101'
        fix['delco'] = '42045'
        fix['steel city'] = '42003'
        fix['queens'] = '36081'
        for nyc_variant in ['nyc', 'new york, new york','new york city', 'new york, ny', 'ny, ny', 'the big apple']:
           fix[nyc_variant] = '36047;36061;36081;36085;36005'

    updated = 0
    filtered_out = 0
    locfields_removed = {}
    if points:
        input_fn = "./{0}/user_points.csv".format(vgi_repository)
    else:
        input_fn = "./{0}/user_counties.csv".format(vgi_repository)
    with open(input_fn, "r") as fin:
        csvreader = csv.reader(fin)
        with open(input_fn.replace(".csv","_cleaned.csv"), "w") as fout:
            csvwriter = csv.writer(fout)
            for line in csvreader:
                if line[1] and line[1].lower().strip() in filter_out:
                    csvwriter.writerow([line[0], line[1], None])
                    if line[1].lower().strip() in locfields_removed:
                        locfields_removed[line[1].lower().strip()] += 1
                    else:
                        locfields_removed[line[1].lower().strip()] = 1
                    filtered_out += 1
                elif line[1].lower().strip() in fix:
                    csvwriter.writerow([line[0], line[1], fix[line[1].lower().strip()]])
                    updated += 1
                else:
                    csvwriter.writerow(line)

    for locfield in locfields_removed:
        print(locfield, locfields_removed[locfield])
    print("{0} updated and {1} filtered out.".format(updated, filtered_out))

if __name__ == "__main__":
    main()<commit_msg>clarify fixes<commit_after>import csv

def main(vgi_repository='t51m', points=True):
    filter_out = {}  # dict for fast look-up
    with open("location_field/state_table.csv", "r") as fin:
        csvreader = csv.reader(fin)
        next(csvreader)
        for line in csvreader:
            if line[1] != 'Washington DC':
                for direction in ['', 'southern ', 'eastern ', 'northern ', 'central ', 'western ']:
                    filter_out[direction + line[1].lower()] = True  # e.g. Alabama
                    filter_out[direction + line[2].lower()] = True  # e.g. AL
                    filter_out[direction + line[1].lower() + ", usa"] = True
                    filter_out[direction + line[2].lower() + ", usa"] = True
                    filter_out[direction + line[1].lower() + ", us"] = True
                    filter_out[direction + line[2].lower() + ", us"] = True

    # Most of these are not necessary - they come from an earlier version that was noisier.
    filter_out['america'] = True
    filter_out['etats-unis'] = True
    filter_out['usa'] = True
    filter_out['u.s.a.'] = True
    filter_out['us'] = True
    filter_out['u.s.'] = True
    filter_out['united states'] = True
    filter_out['united states of america'] = True
    filter_out['estados unidos'] = True

    filter_out['pacific northwest'] = True
    filter_out['the mitten'] = True
    filter_out['tejas'] = True
    filter_out['new england'] = True
    filter_out['lone star state'] = True

    filter_out['earth'] = True
    filter_out['nowhere'] = True
    filter_out['arg'] = True
    filter_out['central city'] = True
    filter_out['location'] = True
    filter_out['disney'] = True
    filter_out['clouds'] = True
    filter_out['area 51'] = True
    filter_out['westside'] = True
    filter_out['lol'] = True
    filter_out['house'] = True
    filter_out['krypton'] = True
    filter_out['pandora'] = True
    filter_out['cosmos'] = True
    filter_out['beach'] = True
    filter_out['happy'] = True
    filter_out['mars'] = True
    filter_out['bed'] = True
    filter_out['wonderland'] = True
    filter_out['south'] = True
    filter_out['nirvana'] = True
    filter_out['bdg'] = True
    filter_out['life'] = True
    filter_out['heart'] = True
    filter_out['indian'] = True
    filter_out['eastern'] = True
    filter_out['mlk'] = True
    filter_out['hope'] = True

    filter_out['badlands'] = True
    filter_out['dixie'] = True
    filter_out['san andreas'] = True
    filter_out['transylvania'] = True
    filter_out['belgique'] = True
    filter_out['pateros'] = True  # Manila
    filter_out['corsica'] = True
    filter_out['wimbledon'] = True
    filter_out['fsu'] = True
    filter_out['scandinavia'] = True
    filter_out['mhs'] = True
    filter_out['queen city'] = True # likely Cincinnati but a general term too...
    filter_out['ayrshire'] = True
    filter_out['alberta'] = True
    filter_out['newfoundland'] = True
    filter_out['bromley'] = True  # district in London

    with open('location_field/country_codes.tsv', 'r') as fin:
        csvreader = csv.reader(fin, delimiter='\t')
        header = next(csvreader)
        country_idx = header.index('Country')
        for line in csvreader:
            country = line[country_idx].lower().strip()
            filter_out[country] = True

    fix = {}
    if points:
        fix['washington dc'] = '(38.89511, -77.03637)'
        fix['twin cities, mn'] = '(44.96219, -93.178555)'  # Average of Minneaplis center and St. Paul center
        fix['twin cities, minnesota'] = '(44.96219, -93.178555)'
        fix['city of angels'] = '(34.05223, -118.24368)'
        fix['the city of angels'] = '(34.05223, -118.24368)'
        fix['phil'] = "(39.95233, -75.16379)"
        fix['delco'] = "(39.9168, -75.3989)"
        fix['steel city'] = "(40.44062, -79.99589)"
        fix['queens'] = "(40.76538, -73.81736)"
        for nyc_variant in ['nyc', 'new york, new york','new york city', 'new york, ny', 'ny, ny', 'the big apple']:
            fix[nyc_variant] = '(40.78343, -73.96625)'

    else:
        fix['washington dc'] = '11001'
        fix['twin cities, mn'] = '27053;27123'
        fix['twin cities, minnesota'] = '27053;27123'
        fix['city of angels'] = '06037'
        fix['the city of angels'] = '06037'
        fix['phil'] = '42101'
        fix['delco'] = '42045'
        fix['steel city'] = '42003'
        fix['queens'] = '36081'
        for nyc_variant in ['nyc', 'new york, new york','new york city', 'new york, ny', 'ny, ny', 'the big apple']:
           fix[nyc_variant] = '36047;36061;36081;36085;36005'

    updated = 0
    filtered_out = 0
    locfields_removed = {}
    if points:
        input_fn = "./{0}/user_points.csv".format(vgi_repository)
    else:
        input_fn = "./{0}/user_counties.csv".format(vgi_repository)
    with open(input_fn, "r") as fin:
        csvreader = csv.reader(fin)
        with open(input_fn.replace(".csv","_cleaned.csv"), "w") as fout:
            csvwriter = csv.writer(fout)
            for line in csvreader:
                if line[1] and line[1].lower().strip() in filter_out:
                    csvwriter.writerow([line[0], line[1], None])
                    if line[1].lower().strip() in locfields_removed:
                        locfields_removed[line[1].lower().strip()] += 1
                    else:
                        locfields_removed[line[1].lower().strip()] = 1
                    filtered_out += 1
                elif line[1].lower().strip() in fix:
                    csvwriter.writerow([line[0], line[1], fix[line[1].lower().strip()]])
                    updated += 1
                else:
                    csvwriter.writerow(line)

    for locfield in locfields_removed:
        print(locfield, locfields_removed[locfield])
    print("{0} updated and {1} filtered out.".format(updated, filtered_out))

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before># Copyright (C) 2013  Matthieu Caneill <matthieu.caneill@gmail.com>
#
# This file is part of Firewose.
#
# Firewose is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# to generate links safely
try:
    from werkzeug.urls import url_quote
except ImportError:
    from urlparse import quote as url_quote

from sqlalchemy.ext.declarative import declarative_base

from debian.debian_support import version_compare

from firewose.lib.orm import *
from sqlalchemy import and_

"""
This module is intented to generate links to point on code listing,
e.g. the one used with Debian on http://sources.debian.net
"""

def get_source_url(url,
                   package, version, release, path,
                   start_line, end_line=None,
                   message=None):
    """
    Creates an url for source code visualisation.

    Args:
    url_prefix: the base url
         e.g.: http://sources.debian.net/src/{package}/{version}/{path}?...
    package: the package name, e.g. python-ethtool
    version: the package version
    release: the package release
    start_line: the first line to highlight and the one pointed by {anchor}
    end_line: if provided, the last line to highlight
    message: if provided, the message (e.g. sources.d.n supports this)
    """
    url = (url
           .replace("{package}", url_quote(package))
           .replace("{version}", url_quote(version))
           .replace("{release}", url_quote(release))
           .replace("{path}", url_quote(path))
           .replace("{anchor}", str(start_line))
           )

    if end_line is not None:
        lines_range = str(start_line) + ":" + str(end_line)
    else:
        lines_range = str(start_line)
    url = url.replace("{lines_range}", lines_range)

    if message is not None:
        url = url.replace("{message}", "%d:error:%s" % (start_line,
                                                        url_quote(message)))
    else:
        url = url.replace("{message}", "") # we don't want {message} of course

    return url

def packages_for_person(session, login):
    """
    Returns a list of package names, whose responsible is login.
    """
    def base_query():
        return (session.query(DebianPackage.name)
                .filter(and_(
                    DebianPackage.name == \
                        DebianPackagePeopleMapping.debian_package_name)))

    if "@" in login: # email
        res = (base_query().filter(
                DebianPackagePeopleMapping.debian_maintainer_email == login)
               .all())
    else: # login or full name
        res = (base_query().filter(
                DebianPackagePeopleMapping.debian_maintainer_email == \
                    DebianMaintainer.email,
                DebianMaintainer.name == login)
               .all())
        if len(res) == 0: # login
            login = login + "@debian.org"
            res = (base_query().filter(
                    DebianPackagePeopleMapping.debian_maintainer_email == login)
               .all())
    return res

###################################################
### Debian specific tables, objects and mappers ###
###################################################

Base = declarative_base()

class DebianPackage(Base):
    __tablename__ = "debian_packages"

    name = Column(String, primary_key=True)

    def __init__(self, name):
        self.name = name

Index('ix_debian_packages_name', DebianPackage.name)

class DebianMaintainer(Base):
    __tablename__ = "debian_maintainers"

    email = Column(String, primary_key=True)
    name = Column('name', String)

    def __init__(self, email, name):
        self.email = email
        self.name = name

Index('ix_debian_maintainers_name', DebianMaintainer.name)

class DebianPackagePeopleMapping(Base):
    __tablename__ = "debian_package_people_mapping"

    debian_package_name = Column(
        String, ForeignKey('debian_packages.name'), primary_key=True)
    debian_maintainer_email = Column(
        String, ForeignKey('debian_maintainers.email'), primary_key=True)

    def __init__(self, debian_package_name, debian_maintainer_email):
        self.debian_package_name = debian_package_name
        self.debian_maintainer_email = debian_maintainer_email

Index('ix_debian_package_people_mapping_maintainer_id',
      DebianPackagePeopleMapping.debian_maintainer_email)
<commit_msg>add: debianutils: functions for filtering by maintainer<commit_after># Copyright (C) 2013  Matthieu Caneill <matthieu.caneill@gmail.com>
#
# This file is part of Firewose.
#
# Firewose is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) any
# later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
# details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# to generate links safely
try:
    from werkzeug.urls import url_quote
except ImportError:
    from urlparse import quote as url_quote

from sqlalchemy.ext.declarative import declarative_base

from debian.debian_support import version_compare

from firewose.lib.orm import *
from firewose.web.app import session
from sqlalchemy import and_

"""
This module is intented to generate links to point on code listing,
e.g. the one used with Debian on http://sources.debian.net
"""

def get_source_url(url,
                   package, version, release, path,
                   start_line, end_line=None,
                   message=None):
    """
    Creates an url for source code visualisation.

    Args:
    url_prefix: the base url
         e.g.: http://sources.debian.net/src/{package}/{version}/{path}?...
    package: the package name, e.g. python-ethtool
    version: the package version
    release: the package release
    start_line: the first line to highlight and the one pointed by {anchor}
    end_line: if provided, the last line to highlight
    message: if provided, the message (e.g. sources.d.n supports this)
    """
    url = (url
           .replace("{package}", url_quote(package))
           .replace("{version}", url_quote(version))
           .replace("{release}", url_quote(release))
           .replace("{path}", url_quote(path))
           .replace("{anchor}", str(start_line))
           )

    if end_line is not None:
        lines_range = str(start_line) + ":" + str(end_line)
    else:
        lines_range = str(start_line)
    url = url.replace("{lines_range}", lines_range)

    if message is not None:
        url = url.replace("{message}", "%d:error:%s" % (start_line,
                                                        url_quote(message)))
    else:
        url = url.replace("{message}", "") # we don't want {message} of course

    return url

def packages_for_person(session, login):
    """
    Returns a list of package names, whose responsible is login.
    """
    def base_query():
        return (session.query(DebianPackage.name)
                .filter(and_(
                    DebianPackage.name == \
                        DebianPackagePeopleMapping.debian_package_name)))

    if "@" in login: # email
        res = (base_query().filter(
                DebianPackagePeopleMapping.debian_maintainer_email == login)
               .all())
    else: # login or full name
        res = (base_query().filter(
                DebianPackagePeopleMapping.debian_maintainer_email == \
                    DebianMaintainer.email,
                DebianMaintainer.name == login)
               .all())
        if len(res) == 0: # login
            login = login + "@debian.org"
            res = (base_query().filter(
                    DebianPackagePeopleMapping.debian_maintainer_email == login)
               .all())
    return res

def email_for_person(person):
    """
    Given a login, a full name, or an email, returns the corresponding email.
    """
    if "@" in person:
        return person
    res = (session.query(DebianMaintainer)
           .filter(DebianMaintainer.name == person)
           .first())
    if res:
        return res.email

    res = (session.query(DebianMaintainer)
           .filter(DebianMaintainer.email == person + "@debian.org")
           .first())
    if res:
        return res.email

    return None

###################################################
### Debian specific tables, objects and mappers ###
###################################################

Base = declarative_base()

class DebianPackage(Base):
    __tablename__ = "debian_packages"

    name = Column(String, primary_key=True, index=True)
    mappings = relationship("DebianPackagePeopleMapping", backref="package")

    def __init__(self, name):
        self.name = name

class DebianMaintainer(Base):
    __tablename__ = "debian_maintainers"

    email = Column(String, primary_key=True, index=True)
    name = Column(String, index=True)
    mappings = relationship("DebianPackagePeopleMapping", backref="maintainer")

    def __init__(self, email, name):
        self.email = email
        self.name = name

class DebianPackagePeopleMapping(Base):
    __tablename__ = "debian_package_people_mapping"

    package_name = Column(
        String, ForeignKey('debian_packages.name'), primary_key=True)
    maintainer_email = Column(
        String, ForeignKey('debian_maintainers.email'),
        primary_key=True, index=True)

    def __init__(self, package_name, maintainer_email):
        self.package = package_name
        self.maintainer = maintainer_email
<|endoftext|>


<commit_before>#!/usr/bin/env python3
import argparse
import logging
import pathlib
import random
import sys
script_dir = pathlib.Path(__file__).resolve().parent
root_dir = script_dir.parent
sys.path.append(str(root_dir))
import dunovo_parsers
from bfx import getreads

USAGE = "%(prog)s [options]"
DESCRIPTION = """"""

def make_argparser():
  parser = argparse.ArgumentParser(add_help=False, description=DESCRIPTION)
  options = parser.add_argument_group('Options')
  parser.add_argument('families', metavar='families.tsv', type=argparse.FileType('r'),
    help='The input reads, sorted into families (the output of make-families.sh).')
  parser.add_argument('fastq1', metavar='reads_1.fq', type=argparse.FileType('r'),
    help='The raw input reads (mate 1).')
  parser.add_argument('fastq2', metavar='reads_2.fq', type=argparse.FileType('r'),
    help='The raw input reads (mate 2).')
  parser.add_argument('fastq1_out', metavar='selected_reads_1.fq', type=argparse.FileType('w'),
    help='Output filename for the subsampled reads (mate 1). '
      'Warning: any existing file will be overwritten.')
  parser.add_argument('fastq2_out', metavar='selected_reads_2.fq', type=argparse.FileType('w'),
    help='Output filename for the subsampled reads (mate 2). '
      'Warning: any existing file will be overwritten.')
  parser.add_argument('-f', '--fraction', type=float, default=1,
    help='Fraction of families to output. Default: %(default)s')
  parser.add_argument('-s', '--seed', type=int,
    help='Random number generator seed. Default: %(default)s')
  parser.add_argument('-P', '--prepended', action='store_true',
    help='The families.tsv file is the result of correct.py --prepend.')
  options.add_argument('-h', '--help', action='help',
    help='Print this argument help text and exit.')
  logs = parser.add_argument_group('Logging')
  logs.add_argument('-l', '--log', type=argparse.FileType('w'), default=sys.stderr,
    help='Print log messages to this file instead of to stderr. Warning: Will overwrite the file.')
  volume = logs.add_mutually_exclusive_group()
  volume.add_argument('-q', '--quiet', dest='volume', action='store_const', const=logging.CRITICAL,
    default=logging.WARNING)
  volume.add_argument('-v', '--verbose', dest='volume', action='store_const', const=logging.INFO)
  volume.add_argument('-D', '--debug', dest='volume', action='store_const', const=logging.DEBUG)
  return parser

def main(argv):

  parser = make_argparser()
  args = parser.parse_args(argv[1:])

  logging.basicConfig(stream=args.log, level=args.volume, format='%(message)s')

  if args.seed is None:
    seed = random.randint(1, 2**32)
    logging.error(f'Seed: {seed}')
  else:
    seed = args.seed
  random.seed(seed)

  logging.info('Info: Reading input families file..')
  families = dunovo_parsers.parse_make_families(args.families, args.prepended)
  chosen_families = choose_elements(families, args.fraction)
  mate1_names, mate2_names = get_all_read_names(chosen_families, args.fraction)
  logging.info('Info: Filtering raw FASTQ into subsampled FASTQ (mate 1)..')
  find_and_write_chosen_reads(mate1_names, args.fastq1, args.fastq1_out)
  logging.info('Info: Filtering raw FASTQ into subsampled FASTQ (mate 2)..')
  find_and_write_chosen_reads(mate2_names, args.fastq2, args.fastq2_out)

def choose_elements(elements, fraction):
  for element in elements:
    if random.random() <= fraction:
      yield element

def get_all_read_names(families, fraction):
  all_mate1_names = set()
  all_mate2_names = set()
  for family in families:
    mate1_names, mate2_names = get_read_names(family)
    all_mate1_names.update(mate1_names)
    all_mate2_names.update(mate2_names)
  return all_mate1_names, all_mate2_names

def get_read_names(family):
  mate1_names = []
  mate2_names = []
  for strand_family in family.ab, family.ba:
    mate1s = strand_family.mate1.reads
    mate1_names.extend([read.name for read in mate1s])
    mate2s = strand_family.mate2.reads
    mate2_names.extend([read.name for read in mate2s])
  return mate1_names, mate2_names

def find_and_write_chosen_reads(chosen_names, input_fastq, output_fastq):
  input_reads = getreads.getparser(input_fastq, filetype='fastq')
  chosen_reads = find_chosen_reads(input_reads, chosen_names)
  write_reads(chosen_reads, output_fastq)

def find_chosen_reads(input_reads, read_names):
  for read in input_reads:
    if read.name in read_names:
      yield read

def write_reads(reads, outfile):
  for read in reads:
    print(read.to_fastq(), file=outfile)

def fail(message):
  logging.critical('Error: '+str(message))
  if __name__ == '__main__':
    sys.exit(1)
  else:
    raise Exception(message)

if __name__ == '__main__':
  try:
    sys.exit(main(sys.argv))
  except BrokenPipeError:
    pass
<commit_msg>utils/subsample.py: Tweaks and adding a description.<commit_after>#!/usr/bin/env python3
import argparse
import logging
import pathlib
import random
import sys
script_dir = pathlib.Path(__file__).resolve().parent
root_dir = script_dir.parent
sys.path.append(str(root_dir))
import dunovo_parsers
from bfx import getreads

DESCRIPTION = """Subsample an input dataset on a per-family basis, not per read."""

def make_argparser():
  parser = argparse.ArgumentParser(add_help=False, description=DESCRIPTION)
  options = parser.add_argument_group('Options')
  parser.add_argument('families', metavar='families.tsv', type=argparse.FileType('r'),
    help='The input reads, sorted into families (the output of make-families.sh).')
  parser.add_argument('fastq1', metavar='reads_1.fq', type=argparse.FileType('r'),
    help='The raw input reads (mate 1).')
  parser.add_argument('fastq2', metavar='reads_2.fq', type=argparse.FileType('r'),
    help='The raw input reads (mate 2).')
  parser.add_argument('fastq1_out', metavar='selected_reads_1.fq', type=argparse.FileType('w'),
    help='Output filename for the subsampled reads (mate 1). '
      'Warning: any existing file will be overwritten.')
  parser.add_argument('fastq2_out', metavar='selected_reads_2.fq', type=argparse.FileType('w'),
    help='Output filename for the subsampled reads (mate 2). '
      'Warning: any existing file will be overwritten.')
  parser.add_argument('-f', '--fraction', type=float, default=1,
    help='Fraction of families to output. Default: %(default)s')
  parser.add_argument('-s', '--seed', type=int,
    help='Random number generator seed. Default: %(default)s')
  parser.add_argument('-P', '--prepended', action='store_true',
    help='The families.tsv file is the result of correct.py --prepend.')
  options.add_argument('-h', '--help', action='help',
    help='Print this argument help text and exit.')
  logs = parser.add_argument_group('Logging')
  logs.add_argument('-l', '--log', type=argparse.FileType('w'), default=sys.stderr,
    help='Print log messages to this file instead of to stderr. Warning: Will overwrite the file.')
  volume = logs.add_mutually_exclusive_group()
  volume.add_argument('-q', '--quiet', dest='volume', action='store_const', const=logging.CRITICAL,
    default=logging.WARNING)
  volume.add_argument('-v', '--verbose', dest='volume', action='store_const', const=logging.INFO)
  volume.add_argument('-D', '--debug', dest='volume', action='store_const', const=logging.DEBUG)
  return parser

def main(argv):

  parser = make_argparser()
  args = parser.parse_args(argv[1:])

  logging.basicConfig(stream=args.log, level=args.volume, format='%(message)s')

  if args.seed is None:
    seed = random.randint(1, 2**32)
    logging.error(f'Seed: {seed}')
  else:
    seed = args.seed
  random.seed(seed)

  logging.info('Info: Reading input families file..')
  families = dunovo_parsers.parse_make_families(args.families, args.prepended)
  chosen_families = choose_elements(families, args.fraction)
  mate1_names, mate2_names = get_all_read_names(chosen_families)
  logging.info('Info: Filtering raw FASTQ into subsampled FASTQ (mate 1)..')
  find_and_write_chosen_reads(mate1_names, args.fastq1, args.fastq1_out)
  logging.info('Info: Filtering raw FASTQ into subsampled FASTQ (mate 2)..')
  find_and_write_chosen_reads(mate2_names, args.fastq2, args.fastq2_out)

def choose_elements(elements, fraction):
  for element in elements:
    if random.random() <= fraction:
      yield element

def get_all_read_names(families):
  all_mate1_names = set()
  all_mate2_names = set()
  for family in families:
    mate1_names, mate2_names = get_read_names(family)
    all_mate1_names.update(mate1_names)
    all_mate2_names.update(mate2_names)
  return all_mate1_names, all_mate2_names

def get_read_names(family):
  mate1_names = []
  mate2_names = []
  for strand_family in family.ab, family.ba:
    mate1s = strand_family.mate1.reads
    mate1_names.extend([read.name for read in mate1s])
    mate2s = strand_family.mate2.reads
    mate2_names.extend([read.name for read in mate2s])
  return mate1_names, mate2_names

def find_and_write_chosen_reads(chosen_names, input_fastq, output_fastq):
  input_reads = getreads.getparser(input_fastq, filetype='fastq')
  chosen_reads = find_chosen_reads(input_reads, chosen_names)
  write_reads(chosen_reads, output_fastq)

def find_chosen_reads(input_reads, read_names):
  for read in input_reads:
    if read.name in read_names:
      yield read

def write_reads(reads, outfile):
  for read in reads:
    print(read.to_fastq(), file=outfile)

def fail(message):
  logging.critical('Error: %s', message)
  if __name__ == '__main__':
    sys.exit(1)
  else:
    raise Exception(message)

if __name__ == '__main__':
  try:
    sys.exit(main(sys.argv))
  except BrokenPipeError:
    pass
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

import codecs
import json
import os
import re
from threading import Lock

try:  # Python 2
    from urllib2 import urlopen, Request, URLError
    from urllib import quote_plus
except ImportError:  # Python 3
    from urllib.request import urlopen, Request
    from urllib.parse import quote_plus
    from urllib.error import URLError

try:
    import socket
    import gevent.socket

    if socket.socket is gevent.socket.socket:
        from gevent import sleep
    else:
        from time import sleep
except ImportError, AttributeError:
    from time import sleep

def get(url):
    with get.lock:
        request = Request(url)

        attempt = 0

        while attempt < settings.HTTP_RETRIES:
            attempt += 1

            try:
                return urlopen(request, timeout=settings.HTTP_TIMEOUT).read()
            except (URLError, OSError):
                if attempt == settings.HTTP_RETRIES:
                    raise FakeUserAgentError
                else:
                    sleep(settings.HTTP_TIMEOUT)
get.lock = Lock()

def get_browsers():
    """
    very very hardcoded/dirty re/split stuff, but no dependencies
    """
    html = get(settings.BROWSERS_STATS_PAGE)
    html = html.decode('windows-1252')
    html = html.split('
')[1]
    html = html.split('
')[0]

    browsers = re.findall(r'\.asp">(.+?)<', html, re.UNICODE)

    browsers = [
        settings.OVERRIDES.get(browser, browser) for browser in browsers
    ]

    browsers_statistics = re.findall(
        r'td\sclass="right">(.+?)\s', html, re.UNICODE
    )

    return list(zip(browsers, browsers_statistics))

def get_browser_versions(browser):
    """
    very very hardcoded/dirty re/split stuff, but no dependencies
    """
    html = get(settings.BROWSER_BASE_PAGE.format(browser=quote_plus(browser)))
    html = html.decode('iso-8859-1')
    html = html.split('
')[1]
    html = html.split('
')[0]

    browsers_iter = re.finditer(r'\?id=\d+\'>(.+?)</a', html, re.UNICODE)

    browsers = []

    for browser in browsers_iter:
        if 'more' in browser.group(1).lower():
            continue

        browsers.append(browser.group(1))

        if len(browsers) == settings.BROWSERS_COUNT_LIMIT:
            break

    return browsers

def load():
    browsers_dict = {}
    randomize_dict = {}

    try:
        for item in get_browsers():
            browser, percent = item

            browser_key = browser

            for value, replacement in settings.REPLACEMENTS.items():
                browser_key = browser_key.replace(value, replacement)

            browser_key = browser_key.lower()

            browsers_dict[browser_key] = get_browser_versions(browser)

            for _ in range(int(float(percent) * 10)):
                randomize_dict[str(len(randomize_dict))] = browser_key
    except Exception:
        server_cached_data = get(settings.CACHE_SERVER)

        try:
            return json.loads(server_cached_data.decode('utf-8'))
        except (UnicodeDecodeError, TypeError, ValueError):
            raise FakeUserAgentError
    else:
        return {
            'browsers': browsers_dict,
            'randomize': randomize_dict
        }

def write(data):
    with codecs.open(settings.DB, encoding='utf-8', mode='wb+',) as fp:
        json.dump(data, fp)

def read():
    with codecs.open(settings.DB, encoding='utf-8', mode='rb',) as fp:
        return json.load(fp)

def exist():
    return os.path.isfile(settings.DB)

def rm():
    if exist():
        os.remove(settings.DB)

def update():
    if exist():
        rm()

    write(load())

def load_cached():
    if not exist():
        update()

    return read()

from fake_useragent import settings  # noqa # isort:skip
from fake_useragent.errors import FakeUserAgentError  # noqa # isort:skip
<commit_msg>Fix Python3+ except statment.<commit_after>from __future__ import absolute_import, unicode_literals

import codecs
import json
import os
import re
from threading import Lock

try:  # Python 2
    from urllib2 import urlopen, Request, URLError
    from urllib import quote_plus
except ImportError:  # Python 3
    from urllib.request import urlopen, Request
    from urllib.parse import quote_plus
    from urllib.error import URLError

try:
    import socket
    import gevent.socket

    if socket.socket is gevent.socket.socket:
        from gevent import sleep
    else:
        from time import sleep
except (ImportError, AttributeError):
    from time import sleep

def get(url):
    with get.lock:
        request = Request(url)

        attempt = 0

        while attempt < settings.HTTP_RETRIES:
            attempt += 1

            try:
                return urlopen(request, timeout=settings.HTTP_TIMEOUT).read()
            except (URLError, OSError):
                if attempt == settings.HTTP_RETRIES:
                    raise FakeUserAgentError
                else:
                    sleep(settings.HTTP_TIMEOUT)
get.lock = Lock()

def get_browsers():
    """
    very very hardcoded/dirty re/split stuff, but no dependencies
    """
    html = get(settings.BROWSERS_STATS_PAGE)
    html = html.decode('windows-1252')
    html = html.split('
')[1]
    html = html.split('
')[0]

    browsers = re.findall(r'\.asp">(.+?)<', html, re.UNICODE)

    browsers = [
        settings.OVERRIDES.get(browser, browser) for browser in browsers
    ]

    browsers_statistics = re.findall(
        r'td\sclass="right">(.+?)\s', html, re.UNICODE
    )

    return list(zip(browsers, browsers_statistics))

def get_browser_versions(browser):
    """
    very very hardcoded/dirty re/split stuff, but no dependencies
    """
    html = get(settings.BROWSER_BASE_PAGE.format(browser=quote_plus(browser)))
    html = html.decode('iso-8859-1')
    html = html.split('
')[1]
    html = html.split('
')[0]

    browsers_iter = re.finditer(r'\?id=\d+\'>(.+?)</a', html, re.UNICODE)

    browsers = []

    for browser in browsers_iter:
        if 'more' in browser.group(1).lower():
            continue

        browsers.append(browser.group(1))

        if len(browsers) == settings.BROWSERS_COUNT_LIMIT:
            break

    return browsers

def load():
    browsers_dict = {}
    randomize_dict = {}

    try:
        for item in get_browsers():
            browser, percent = item

            browser_key = browser

            for value, replacement in settings.REPLACEMENTS.items():
                browser_key = browser_key.replace(value, replacement)

            browser_key = browser_key.lower()

            browsers_dict[browser_key] = get_browser_versions(browser)

            for _ in range(int(float(percent) * 10)):
                randomize_dict[str(len(randomize_dict))] = browser_key
    except Exception:
        server_cached_data = get(settings.CACHE_SERVER)

        try:
            return json.loads(server_cached_data.decode('utf-8'))
        except (UnicodeDecodeError, TypeError, ValueError):
            raise FakeUserAgentError
    else:
        return {
            'browsers': browsers_dict,
            'randomize': randomize_dict
        }

def write(data):
    with codecs.open(settings.DB, encoding='utf-8', mode='wb+',) as fp:
        json.dump(data, fp)

def read():
    with codecs.open(settings.DB, encoding='utf-8', mode='rb',) as fp:
        return json.load(fp)

def exist():
    return os.path.isfile(settings.DB)

def rm():
    if exist():
        os.remove(settings.DB)

def update():
    if exist():
        rm()

    write(load())

def load_cached():
    if not exist():
        update()

    return read()

from fake_useragent import settings  # noqa # isort:skip
from fake_useragent.errors import FakeUserAgentError  # noqa # isort:skip
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import math
import os
import sys

SUBMIT_FILE = '''
executable     = run_sim.sh
universe       = vanilla

Error   = log/err.$(Cluster).$(Process)
Output  = log/out.$(Cluster).$(Process)
Log     = log/log.$(Cluster).$(Process)

Requirements = (HAS_CVMFS_xenon_opensciencegrid_org ) && \\
               (OpSysAndVer =?= "CentOS6" || OpSysAndVer =?= "RedHat6" || OpSysAndVer =?= "SL6") && \\
               (GLIDEIN_ResourceName =!= "BNL-ATLAS") && \\
               (GLIDEIN_ResourceName =!= "AGLT2")
+WantExperimental = True
+WANT_RCC_ciconnect = True

transfer_executable = True
transfer_output_files = results
when_to_transfer_output = ON_EXIT
arguments = $(flavor) $(config) $(mc_version) $(events) $(pax_version)
queue 1

'''

MC_FLAVORS = ('G4', 'NEST', 'G4p10')
CONFIGS = ('TPC_Kr83m', 'TPC_Kr85', 'WholeLXe_Rn220', 'WholeLXe_Rn222')
MC_PATH = '/cvmfs/xenon.opensciencegrid.org/releases/mc/'
PAX_PATH = "/cvmfs/xenon.opensciencegrid.org/releases/anaconda/2.4/envs/"

def get_mc_versions():
    """
    Return a tuple with mc versions that are available

    :return: tuple with string of mc versions available
    """
    try:
        if os.path.isdir(MC_PATH):
            versions = os.listdir(MC_PATH)
            versions.sort()
            return tuple(versions)
        return ()
    except OSError:
        sys.stderr.write("Can't get mc versions from {0}\n".format(MC_PATH))
        return ()

def get_pax_versions():
    """
    Return a tuple with pax versions that are available

    :return: tuple with string of mc versions available
    """
    try:
        versions = []
        if not os.path.isdir(PAX_PATH):
            return ()
        for entry in os.listdir(PAX_PATH):
            if entry.startswith('pax_'):
                versions.append(entry.replace('pax_', ''))
        versions.sort()
        return tuple(versions)
    except OSError:
        sys.stderr.write("Can't get pax versions from {0}\n".format(PAX_PATH))
        return ()

MC_VERSIONS = get_mc_versions()
PAX_VERSIONS = get_pax_versions()

def run_main():
    """
    Main function to process user input and then generate the appropriate submit files

    :return: exit code -- 0 on success, 1 otherwise
    """

    parser = argparse.ArgumentParser(description="Create a set of files for doing MC simulation for X1T")

    parser.add_argument('--flavor', dest='mc_flavor',
                        action='store', required=True,
                        choices=MC_FLAVORS,
                        help='MC flavor to use')
    parser.add_argument('--config', dest='mc_config',
                        action='store', required=True,
                        choices=CONFIGS,
                        help='configuration to use')
    parser.add_argument('--events', dest='num_events',
                        action='store', required=True,
                        type=int,
                        help='number of events to generate')
    parser.add_argument('--batch-size', dest='batch_size',
                        action='store', default=2000, type=int,
                        help='max number of events to generate per job '
                             '(default is 2000)')
    parser.add_argument('--mc-version', dest='mc_version',
                        choices=MC_VERSIONS,
                        action='store', required=True,
                        help='version of MC code to use')
    parser.add_argument('--pax-version', dest='pax_version',
                        choices=PAX_VERSIONS,
                        action='store', required=True,
                        help='version of pax to use')
    parser.add_argument('--dagfile-file', dest='dag_file',
                        action='store', default='mc.dag',
                        help='file to write dag to')
    parser.add_argument('--submit-file', dest='submit_file',
                        action='store', default='mc.submit',
                        help='name of submit to write out')

    args = parser.parse_args(sys.argv[1:])
    if args.num_events == 0:
        sys.stdout.write("No events to generate, exiting")
        return 0
    num_jobs = int(math.ceil(args.num_events / float(args.batch_size)))
    sys.stdout.write("Generating {0} events ".format(args.num_events) +
                     "using {0} jobs\n".format(num_jobs))

    if os.path.exists(args.submit_file):
        sys.stderr.write("Submit file at {0} ".format(args.submit_file) +
                         "already exists, exiting!\n")
        return 1

    if os.path.exists(args.dag_file):
        sys.stderr.write("DAG file at {0} ".format(args.dag_file) +
                         "already exists, exiting!\n")
        return 1

    with open(args.dag_file, 'wt') as dag_file:
        for job in range(0, num_jobs):
            dag_file.write("JOB MC.{0} {1}\n".format(job, args.submit_file))
            dag_file.write('VARS MC.{0} flavor="{1}" '.format(job, args.mc_flavor))
            dag_file.write('config="{0}" '.format(args.mc_config))
            dag_file.write('pax_version="{0}" '.format(args.pax_version))
            dag_file.write('mc_version="{0}" '.format(args.mc_version))
            dag_file.write('events="{0}" '.format(args.num_events))
            dag_file.write("\n")
            dag_file.write("Retry XENON.{0} 3\n".format(job))
    with open(args.submit_file, 'wt') as submit_file:
        submit_file.write(SUBMIT_FILE)

    if not os.path.exists('output'):
        os.mkdir('output')
    if not os.path.exists('log'):
        os.mkdir('log')
    sys.stdout.write("Generated DAG file at {0} ".format(args.dag_file) +
                     "and submit file at {0}, run\n".format(args.submit_file) +
                     "\t\t\tcondor_submit_dag {0}\n".format(args.dag_file) +
                     "to submit.\n")
    return 0

if __name__ == '__main__':
    sys.exit(run_main())
<commit_msg>Fix submit file<commit_after>#!/usr/bin/env python

import argparse
import math
import os
import sys

SUBMIT_FILE = '''
executable     = run_sim.sh
universe       = vanilla

Error   = log/err.$(Cluster).$(Process)
Output  = log/out.$(Cluster).$(Process)
Log     = log/log.$(Cluster).$(Process)

Requirements = (HAS_CVMFS_xenon_opensciencegrid_org ) && \\
               (OpSysAndVer =?= "CentOS6" || OpSysAndVer =?= "RedHat6" || OpSysAndVer =?= "SL6") && \\
               (GLIDEIN_ResourceName =!= "BNL-ATLAS") && \\
               (GLIDEIN_ResourceName =!= "AGLT2")
+WantExperimental = True
+WANT_RCC_ciconnect = True

transfer_executable = True
transfer_output_files = output
when_to_transfer_output = ON_EXIT
arguments = $(flavor) $(config) $(mc_version) $(events) $(pax_version)
queue 1

'''

MC_FLAVORS = ('G4', 'NEST', 'G4p10')
CONFIGS = ('TPC_Kr83m', 'TPC_Kr85', 'WholeLXe_Rn220', 'WholeLXe_Rn222')
MC_PATH = '/cvmfs/xenon.opensciencegrid.org/releases/mc/'
PAX_PATH = "/cvmfs/xenon.opensciencegrid.org/releases/anaconda/2.4/envs/"

def get_mc_versions():
    """
    Return a tuple with mc versions that are available

    :return: tuple with string of mc versions available
    """
    try:
        if os.path.isdir(MC_PATH):
            versions = os.listdir(MC_PATH)
            versions.sort()
            return tuple(versions)
        return ()
    except OSError:
        sys.stderr.write("Can't get mc versions from {0}\n".format(MC_PATH))
        return ()

def get_pax_versions():
    """
    Return a tuple with pax versions that are available

    :return: tuple with string of mc versions available
    """
    try:
        versions = []
        if not os.path.isdir(PAX_PATH):
            return ()
        for entry in os.listdir(PAX_PATH):
            if entry.startswith('pax_'):
                versions.append(entry.replace('pax_', ''))
        versions.sort()
        return tuple(versions)
    except OSError:
        sys.stderr.write("Can't get pax versions from {0}\n".format(PAX_PATH))
        return ()

MC_VERSIONS = get_mc_versions()
PAX_VERSIONS = get_pax_versions()

def run_main():
    """
    Main function to process user input and then generate the appropriate submit files

    :return: exit code -- 0 on success, 1 otherwise
    """

    parser = argparse.ArgumentParser(description="Create a set of files for doing MC simulation for X1T")

    parser.add_argument('--flavor', dest='mc_flavor',
                        action='store', required=True,
                        choices=MC_FLAVORS,
                        help='MC flavor to use')
    parser.add_argument('--config', dest='mc_config',
                        action='store', required=True,
                        choices=CONFIGS,
                        help='configuration to use')
    parser.add_argument('--events', dest='num_events',
                        action='store', required=True,
                        type=int,
                        help='number of events to generate')
    parser.add_argument('--batch-size', dest='batch_size',
                        action='store', default=2000, type=int,
                        help='max number of events to generate per job '
                             '(default is 2000)')
    parser.add_argument('--mc-version', dest='mc_version',
                        choices=MC_VERSIONS,
                        action='store', required=True,
                        help='version of MC code to use')
    parser.add_argument('--pax-version', dest='pax_version',
                        choices=PAX_VERSIONS,
                        action='store', required=True,
                        help='version of pax to use')
    parser.add_argument('--dagfile-file', dest='dag_file',
                        action='store', default='mc.dag',
                        help='file to write dag to')
    parser.add_argument('--submit-file', dest='submit_file',
                        action='store', default='mc.submit',
                        help='name of submit to write out')

    args = parser.parse_args(sys.argv[1:])
    if args.num_events == 0:
        sys.stdout.write("No events to generate, exiting")
        return 0
    num_jobs = int(math.ceil(args.num_events / float(args.batch_size)))
    sys.stdout.write("Generating {0} events ".format(args.num_events) +
                     "using {0} jobs\n".format(num_jobs))

    if os.path.exists(args.submit_file):
        sys.stderr.write("Submit file at {0} ".format(args.submit_file) +
                         "already exists, exiting!\n")
        return 1

    if os.path.exists(args.dag_file):
        sys.stderr.write("DAG file at {0} ".format(args.dag_file) +
                         "already exists, exiting!\n")
        return 1

    with open(args.dag_file, 'wt') as dag_file:
        for job in range(0, num_jobs):
            dag_file.write("JOB MC.{0} {1}\n".format(job, args.submit_file))
            dag_file.write('VARS MC.{0} flavor="{1}" '.format(job, args.mc_flavor))
            dag_file.write('config="{0}" '.format(args.mc_config))
            dag_file.write('pax_version="{0}" '.format(args.pax_version))
            dag_file.write('mc_version="{0}" '.format(args.mc_version))
            dag_file.write('events="{0}" '.format(args.num_events))
            dag_file.write("\n")
            dag_file.write("Retry XENON.{0} 3\n".format(job))
    with open(args.submit_file, 'wt') as submit_file:
        submit_file.write(SUBMIT_FILE)

    if not os.path.exists('output'):
        os.mkdir('output')
    if not os.path.exists('log'):
        os.mkdir('log')
    sys.stdout.write("Generated DAG file at {0} ".format(args.dag_file) +
                     "and submit file at {0}, run\n".format(args.submit_file) +
                     "\t\t\tcondor_submit_dag {0}\n".format(args.dag_file) +
                     "to submit.\n")
    return 0

if __name__ == '__main__':
    sys.exit(run_main())
<|endoftext|>