<commit_before>#encoding=utf-8
import os
import pandas as pd
import numpy as np
import sys
import codecs
import glob
import click
import charset
import make_sum_data
from pyltp import SentenceSplitter

def write_list_to_file(d, file_path, verb=True):
  assert type(d) == list
  if os.path.exists(os.path.dirname(file_path)) == False:
    os.makedirs(os.path.dirname(file_path))
  f = codecs.open(file_path, "w", "utf-8")
  for x in d:
    x = str(x)
    f.write(x.strip() + "\n")
  f.close()
  if verb:
    print("write to {}".format(file_path))

@click.group()
def cli():
  pass

@click.command()
@click.argument("vocab_path")
@click.argument("save_path")
# @click.argument("--del_num/--no-del_num", default=True)
# @click.argument("--del_en/--no-del_en", default=True)
def filter_vocab(vocab_path, save_path, del_num=True, del_en=True):
  f = codecs.open(vocab_path, "r", "utf-8")
  fout = codecs.open(save_path, "w", "utf-8")
  word_sets = set()
  while True:
    line = f.readline()
    if line == "":
      break
    try:
      t = line.strip().split(" ")
      word, count = t[0], t[1]
    except Exception:
      import traceback
      traceback.print_exc()
      print(line)
      continue
    skip = False

    for index, char in enumerate(word):
        if charset.is_alphabet(char) or charset.is_number(char):
          print(word)
          skip = True
          break
        if charset.is_chinese_punctuation(char): ##solve 。榜样
          if len(word) > 1:
            print(word)
            skip = True
            break

    if skip is False:
      wst = "\t".join([word, str(count)]) + "\n"
      fout.write(wst)
  f.close()
  fout.close()

@click.command()
@click.argument("path")
@click.argument("stat_path")
@click.argument("save_path", type=str)
@click.option("--min_abs_chars", type=int, default=0, help="abs min char[0]")
@click.option("--max_abs_chars", type=int, default=int(2e5), help="max abs char[2e5]")
@click.option("--title_index", type=int, default=1, help="title index in one line[1]")
@click.option("--abs_index", type=int, default=2, help="abs index in one line[1]")
@click.option("--article_index", type=int, default=3, help="article index in one line[1]")
@click.option("--cancel_unk_label", is_flag=True, help="whether cancel null label")
@click.option("--abs_max_cov_sents", type=int, default=None, help="abstract very high sim to article 's max sents[100]")
def filter_stat(path, stat_path, save_path, label_index=0, title_index=1, abs_index=2, article_index=3,
                min_abs_chars=0, max_abs_chars=int(2e5), abs_max_cov_sents=100, cancel_unk_label=False):
  len_stat = []
  use_samples =  0
  fin = codecs.open(path, "r", "utf-8")
  fo = codecs.open(save_path, "w", "utf-8")
  total_abs_sent_num , cov_abs_sent_num = 0 , 0
  cov_skip_abs = 0
  i = 0
  no_label_num = 0
  abs_len_filter_num = 0
  while True:
    line = fin.readline()
    if not line:
      break
    if (i+1) % int(1e4) == 0:
      print("finished {}...".format(i))
    i += 1
    t = line.strip().split("\t")
    try:
      label = None
      if label_index is not None:
        label = t[label_index]
      title = t[title_index]
      abstract = t[abs_index]
      article = t[article_index]
    except IndexError:
      continue
    abstract = make_sum_data.preprocess_abs_text(abstract)
    if len(abstract) > max_abs_chars or len(abstract) < min_abs_chars:
      abs_len_filter_num += 1
      continue
    abs_sents = SentenceSplitter.split(abstract)
    total_abs_sent_num += len(abs_sents)
    cov_num = 0
    for sent in abs_sents:
      if sent in article:
        cov_num += 1
    labeled = False
    if label is not None and "null" in label:
      labeled = True
      no_label_num +=1
    cov_abs_sent_num += cov_num
    if len(abs_sents) == cov_num or cov_num > abs_max_cov_sents:
      cov_skip_abs += 1
      continue
    if cancel_unk_label is False or labeled is True:
      use_samples += 1
      len_stat.append([len(article), len(abstract)])
      fo.write("\t".join([title, abstract, article]) + "\n")

  print("filter docs with abs len not in range[{},{})".format(abs_len_filter_num))
  print("skip {} docs with high coverage in article".format(cov_skip_abs))
  print("{} docs have no label".format(no_label_num))
  print("use:all = {}:{}".format(use_samples, i))
  print("abs-cov:all = {}:{}".format(cov_abs_sent_num, total_abs_sent_num))
  len_df = pd.DataFrame(len_stat, columns=["art_len", "abs_len"])
  print("use sample len stat")
  print(len_df.describe())
  len_df.to_csv(stat_path, index=False)
  fo.close()

cli.add_command(filter_vocab)
cli.add_command(filter_stat)

if __name__ == "__main__":
  cli()

<commit_msg>filter article<commit_after>#encoding=utf-8
import os
import pandas as pd
import numpy as np
import sys
import codecs
import glob
import click
import charset
import make_sum_data
from pyltp import SentenceSplitter

def write_list_to_file(d, file_path, verb=True):
  assert type(d) == list
  if os.path.exists(os.path.dirname(file_path)) == False:
    os.makedirs(os.path.dirname(file_path))
  f = codecs.open(file_path, "w", "utf-8")
  for x in d:
    x = str(x)
    f.write(x.strip() + "\n")
  f.close()
  if verb:
    print("write to {}".format(file_path))

@click.group()
def cli():
  pass

@click.command()
@click.argument("vocab_path")
@click.argument("save_path")
# @click.argument("--del_num/--no-del_num", default=True)
# @click.argument("--del_en/--no-del_en", default=True)
def filter_vocab(vocab_path, save_path, del_num=True, del_en=True):
  f = codecs.open(vocab_path, "r", "utf-8")
  fout = codecs.open(save_path, "w", "utf-8")
  word_sets = set()
  while True:
    line = f.readline()
    if line == "":
      break
    try:
      t = line.strip().split(" ")
      word, count = t[0], t[1]
    except Exception:
      import traceback
      traceback.print_exc()
      print(line)
      continue
    skip = False

    for index, char in enumerate(word):
        if charset.is_alphabet(char) or charset.is_number(char):
          print(word)
          skip = True
          break
        if charset.is_chinese_punctuation(char): ##solve 。榜样
          if len(word) > 1:
            print(word)
            skip = True
            break

    if skip is False:
      wst = "\t".join([word, str(count)]) + "\n"
      fout.write(wst)
  f.close()
  fout.close()

@click.command()
@click.argument("path")
@click.argument("stat_path")
@click.argument("save_path", type=str)
@click.option("--min_abs_chars", type=int, default=0, help="abs min char[0]")
@click.option("--max_abs_chars", type=int, default=int(2e5), help="max abs char[2e5]")
@click.option("--title_index", type=int, default=1, help="title index in one line[1]")
@click.option("--abs_index", type=int, default=2, help="abs index in one line[1]")
@click.option("--article_index", type=int, default=3, help="article index in one line[1]")
@click.option("--cancel_unk_label", is_flag=True, help="whether cancel null label")
@click.option("--abs_max_cov_sents", type=int, default=100, help="abstract very high sim to article 's max sents[100]")
def filter_stat(path, stat_path, save_path, label_index=0, title_index=1, abs_index=2, article_index=3,
                min_abs_chars=0, max_abs_chars=int(2e5), abs_max_cov_sents=100, cancel_unk_label=False):
  len_stat = []
  use_samples =  0
  fin = codecs.open(path, "r", "utf-8")
  fo = codecs.open(save_path, "w", "utf-8")
  total_abs_sent_num , cov_abs_sent_num = 0 , 0
  cov_skip_abs = 0
  i = 0
  no_label_num = 0
  abs_len_filter_num = 0
  while True:
    line = fin.readline()
    if not line:
      break
    if (i+1) % int(1e4) == 0:
      print("finished {}...".format(i))
    i += 1
    t = line.strip().split("\t")
    try:
      label = None
      if label_index is not None:
        label = t[label_index]
      title = t[title_index]
      abstract = t[abs_index]
      article = t[article_index]
    except IndexError:
      continue
    abstract = make_sum_data.preprocess_abs_text(abstract)
    if len(abstract) > max_abs_chars or len(abstract) < min_abs_chars:
      abs_len_filter_num += 1
      continue
    abs_sents = SentenceSplitter.split(abstract)
    total_abs_sent_num += len(abs_sents)
    cov_num = 0
    for sent in abs_sents:
      if sent in article:
        cov_num += 1
    labeled = False
    if label is not None and "null" in label:
      labeled = True
      no_label_num +=1
    cov_abs_sent_num += cov_num
    if len(abs_sents) == cov_num or cov_num > abs_max_cov_sents:
      cov_skip_abs += 1
      continue
    if cancel_unk_label is False or labeled is True:
      use_samples += 1
      len_stat.append([len(article), len(abstract)])
      fo.write("\t".join([title, abstract, article]) + "\n")

  print("filter docs with abs len not in range[{},{})".format(abs_len_filter_num))
  print("skip {} docs with high coverage in article".format(cov_skip_abs))
  print("{} docs have no label".format(no_label_num))
  print("use:all = {}:{}".format(use_samples, i))
  print("abs-cov:all = {}:{}".format(cov_abs_sent_num, total_abs_sent_num))
  len_df = pd.DataFrame(len_stat, columns=["art_len", "abs_len"])
  print("use sample len stat")
  print(len_df.describe())
  len_df.to_csv(stat_path, index=False)
  fo.close()

cli.add_command(filter_vocab)
cli.add_command(filter_stat)

if __name__ == "__main__":
  cli()

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# system
import os
import sys
dir = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]
sys.path.append(os.path.join(dir, 'scripts'))

# testing
import mock
import unittest
from mock import patch

# program
import config.load as Config
import config.database as DB
import unosat_flood_portal_collect.collect as Collect

#
# Global variables.
#
TEST_DATA = 'test_flood_portal_output.json'

class CheckCollectorFunctions(unittest.TestCase):
  '''Unit tests checking if the collector is working as expected.'''

  def test_wrapper_doesnt_fail(self):
    assert Collect.Main() != False

  def test_fetch_data_function(self):
    assert Collect.FetchData(url='http://localhost:8080') == False

  def test_processing_works(self):
    data = Collect.DownloadAndProcessData()
    assert type(data) == list

  def test_clean_table_fails(self):
    assert Collect.CleanTable('foo') == False
<commit_msg>adding test for checkg if reading all records work<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# system
import os
import sys
dir = os.path.split(os.path.split(os.path.realpath(__file__))[0])[0]
sys.path.append(os.path.join(dir, 'scripts'))

# testing
import mock
import unittest
from mock import patch

# program
import config.load as Config
import config.database as DB
import utilities.db as Database
import unosat_flood_portal_collect.collect as Collect

#
# Global variables.
#
TEST_DATA = 'test_flood_portal_output.json'

class CheckCollectorFunctions(unittest.TestCase):
  '''Unit tests checking if the collector is working as expected.'''

  def test_wrapper_doesnt_fail(self):
    assert Collect.Main() != False

  def test_fetch_data_function(self):
    assert Collect.FetchData(url='http://localhost:8080') == False

  def test_processing_works(self):
    data = Collect.DownloadAndProcessData()
    assert type(data) == list

  def test_clean_table_fails(self):
    assert Collect.CleanTable('foo') == False

class CheckPatches(unittest.TestCase):
  '''Unit tests that check if the patches are doing what they are supposed to do.'''

  def test_read_all_records_works(self):
    assert Database.ReadAllRecords('unprocessed_data') != False
<|endoftext|>


<commit_before>#encoding=utf-8
import os
import pandas as pd
import numpy as np
import sys
import codecs
import glob
import click
import make_sum_data
from pyltp import SentenceSplitter

def write_list_to_file(d, file_path, verb=True):
  assert type(d) == list
  if os.path.exists(os.path.dirname(file_path)) == False:
    os.makedirs(os.path.dirname(file_path))
  f = codecs.open(file_path, "w", "utf-8")
  for x in d:
    x = str(x)
    f.write(x.strip() + "\n")
  f.close()
  if verb:
    print("write to {}".format(file_path))

@click.command()
@click.argument("path")
@click.argument("stat_path")
@click.argument("save_path", type=str)
@click.option("--title_index", type=int, default=1, help="title index in one line[1]")
@click.option("--abs_index", type=int, default=2, help="abs index in one line[1]")
@click.option("--article_index", type=int, default=3, help="article index in one line[1]")
def filter_stat(path, stat_path, save_path, label_index=None, title_index=1, abs_index=2, article_index=3, abs_max_cov_sents=1):
  len_stat = []
  use_samples =  0
  fin = codecs.open(path, "r", "utf-8")
  fo = codecs.open(save_path, "w", "utf-8")
  total_abs_sent_num , cov_abs_sent_num = 0 , 0
  cov_skip_abs = 0
  i = 0
  no_label_num = 0
  while True:
    line = fin.readline()
    if not line:
      break
    if (i+1) % int(1e4) == 0:
      print("finished {}...".format(i))
    i += 1
    t = line.strip().split("\t")
    try:
      label = None
      if label_index is not None:
        label = t[label]
      title = t[title_index]
      abstract = t[abs_index]
      article = t[article_index]
    except IndexError:
      continue
    abstract = make_sum_data.preprocess_abs_text(abstract)
    abs_sents = SentenceSplitter.split(abstract)
    len_stat.append( [len(article), len(abstract)] )
    total_abs_sent_num += len(abs_sents)
    cov_num = 0
    for sent in abs_sents:
      if sent in article:
        cov_num += 1
    if label is not None and "null" in label:
      no_label_num +=1
    cov_abs_sent_num += cov_num
    if len(abs_sents) == cov_num or cov_num > abs_max_cov_sents:
      cov_skip_abs += 1
      continue
    if label is None or "null" not in label:
      use_samples += 1
      fo.write("\t".join([title, abstract, article]) + "\n")

  print("skip {} docs with high coverage in article".format(cov_skip_abs))
  print("{} docs have no label".format(no_label_num))
  print("use:all = {}:{}".format(use_samples, i))
  print("abs-cov:all = {}:{}".format(cov_abs_sent_num, total_abs_sent_num))
  len_df = pd.DataFrame(len_stat, columns=["art_len", "abs_len"])
  print(len_df.describe())
  len_df.to_csv(stat_path, index=False)

  fo.close()
if __name__ == "__main__":
  filter_stat()

<commit_msg>add stat<commit_after>#encoding=utf-8
import os
import pandas as pd
import numpy as np
import sys
import codecs
import glob
import click
import make_sum_data
from pyltp import SentenceSplitter

def write_list_to_file(d, file_path, verb=True):
  assert type(d) == list
  if os.path.exists(os.path.dirname(file_path)) == False:
    os.makedirs(os.path.dirname(file_path))
  f = codecs.open(file_path, "w", "utf-8")
  for x in d:
    x = str(x)
    f.write(x.strip() + "\n")
  f.close()
  if verb:
    print("write to {}".format(file_path))

@click.command()
@click.argument("path")
@click.argument("stat_path")
@click.argument("save_path", type=str)
@click.option("--title_index", type=int, default=1, help="title index in one line[1]")
@click.option("--abs_index", type=int, default=2, help="abs index in one line[1]")
@click.option("--article_index", type=int, default=3, help="article index in one line[1]")
def filter_stat(path, stat_path, save_path, label_index=0, title_index=1, abs_index=2, article_index=3, abs_max_cov_sents=1):
  len_stat = []
  use_samples =  0
  fin = codecs.open(path, "r", "utf-8")
  fo = codecs.open(save_path, "w", "utf-8")
  total_abs_sent_num , cov_abs_sent_num = 0 , 0
  cov_skip_abs = 0
  i = 0
  no_label_num = 0
  while True:
    line = fin.readline()
    if not line:
      break
    if (i+1) % int(1e4) == 0:
      print("finished {}...".format(i))
    i += 1
    t = line.strip().split("\t")
    try:
      label = None
      if label_index is not None:
        label = t[label]
      title = t[title_index]
      abstract = t[abs_index]
      article = t[article_index]
    except IndexError:
      continue
    abstract = make_sum_data.preprocess_abs_text(abstract)
    abs_sents = SentenceSplitter.split(abstract)
    len_stat.append( [len(article), len(abstract)] )
    total_abs_sent_num += len(abs_sents)
    cov_num = 0
    for sent in abs_sents:
      if sent in article:
        cov_num += 1
    if label is not None and "null" in label:
      no_label_num +=1
    cov_abs_sent_num += cov_num
    if len(abs_sents) == cov_num or cov_num > abs_max_cov_sents:
      cov_skip_abs += 1
      continue
    if label is None or "null" not in label:
      use_samples += 1
      fo.write("\t".join([title, abstract, article]) + "\n")

  print("skip {} docs with high coverage in article".format(cov_skip_abs))
  print("{} docs have no label".format(no_label_num))
  print("use:all = {}:{}".format(use_samples, i))
  print("abs-cov:all = {}:{}".format(cov_abs_sent_num, total_abs_sent_num))
  len_df = pd.DataFrame(len_stat, columns=["art_len", "abs_len"])
  print(len_df.describe())
  len_df.to_csv(stat_path, index=False)

  fo.close()
if __name__ == "__main__":
  filter_stat()

<|endoftext|>


<commit_before>import bloop
import bloop.condition
import pytest
import uuid

def test_duplicate_name_refs(renderer, User):
    """ name refs are re-used for the same name """
    assert renderer.name_ref(User.age) == renderer.name_ref(User.age) == "#n0"

def test_no_refs(renderer):
    """
    when name/value refs are missing, ExpressionAttributeNames/Values
    aren't populated """
    condition = bloop.condition.And()
    expected = {"ConditionExpression": "()"}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_condition_ops(User):
    age, name = (User.age >= 3), (User.name == "foo")

    and_condition = age & name
    assert and_condition.conditions == [age, name]
    assert isinstance(and_condition, bloop.condition.And)

    or_condition = age | name
    assert or_condition.conditions == [age, name]
    assert isinstance(or_condition, bloop.condition.Or)

    not_condition = ~age
    assert not_condition.condition is age
    assert isinstance(not_condition, bloop.condition.Not)

    assert len(age) == len(name) == 1

def test_condition_len(User):
    age, name = (User.age >= 3), (User.name == "foo")
    and_condition = age & name
    or_condition = bloop.condition.And(age, name, age)
    not_condition = ~age

    assert len(or_condition) == 3
    assert len(and_condition) == 2
    assert len(age) == len(name) == len(not_condition) == 1

def test_multi_shortcut(renderer, User):
    """ And or Or with single conditions render as their sole condition """
    age = User.age >= 3
    condition = bloop.condition.And(age)
    expected = {"ConditionExpression": "(#n0 >= :v1)",
                "ExpressionAttributeNames": {"#n0": "age"},
                "ExpressionAttributeValues": {":v1": {"N": "3"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_and_appends(renderer, User):
    """
    ((condition & condition) & condition) flattens the AND into one condition
    """
    age = User.age >= 3
    name = User.name == "foo"
    email = User.email != "bar"
    conditions = [age, name, email]

    condition = (age & name) & email
    assert condition.conditions == conditions

    condition = bloop.condition.Condition()
    for c in conditions:
        condition &= c
    assert condition.conditions == conditions

def test_or_appends(renderer, User):
    """
    ((condition | condition) | condition) flattens the OR into one condition
    """
    age = User.age >= 3
    name = User.name == "foo"
    email = User.email != "bar"
    conditions = [age, name, email]

    condition = (age | name) | email
    assert condition.conditions == conditions

    condition = bloop.condition.Condition()
    for c in conditions:
        condition |= c
    assert condition.conditions == conditions

def test_not(renderer, User):
    age = ~(User.age >= 3)
    condition = bloop.condition.And(age)
    expected = {"ConditionExpression": "(NOT (#n0 >= :v1))",
                "ExpressionAttributeNames": {"#n0": "age"},
                "ExpressionAttributeValues": {":v1": {"N": "3"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_invalid_comparator(User):
    with pytest.raises(ValueError):
        bloop.condition.Comparison(User.age, "foo", 5)

def test_attribute_exists(User, renderer):
    exists = User.age.is_not(None)
    expected_exists = {"ConditionExpression": "(attribute_exists(#n0))",
                       "ExpressionAttributeNames": {"#n0": "age"}}

    renderer.render(exists, "condition")
    assert renderer.rendered == expected_exists

def test_attribute_not_exists(User, renderer):
    not_exists = User.age.is_(None)
    expected_not_exists = {
        "ConditionExpression": "(attribute_not_exists(#n0))",
        "ExpressionAttributeNames": {"#n0": "age"}}

    renderer.render(not_exists, "condition")
    assert renderer.rendered == expected_not_exists

def test_begins_with(renderer, User):
    condition = User.name.begins_with("foo")
    expected = {"ConditionExpression": "(begins_with(#n0, :v1))",
                "ExpressionAttributeNames": {"#n0": "name"},
                "ExpressionAttributeValues": {":v1": {"S": "foo"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_contains(renderer, User):
    condition = User.name.contains("foo")
    expected = {"ConditionExpression": "(contains(#n0, :v1))",
                "ExpressionAttributeNames": {"#n0": "name"},
                "ExpressionAttributeValues": {":v1": {"S": "foo"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_between(renderer, User):
    condition = User.name.between("bar", "foo")
    expected = {"ConditionExpression": "(#n0 BETWEEN :v1 AND :v2)",
                "ExpressionAttributeNames": {"#n0": "name"},
                "ExpressionAttributeValues": {":v1": {"S": "bar"},
                                              ":v2": {"S": "foo"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_in(renderer, User):
    condition = User.name.in_(["bar", "foo"])
    expected = {"ConditionExpression": "(#n0 IN (:v1, :v2))",
                "ExpressionAttributeNames": {"#n0": "name"},
                "ExpressionAttributeValues": {":v1": {"S": "bar"},
                                              ":v2": {"S": "foo"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_base_condition(renderer, User):
    """ (Condition() OP condition) is condition """
    base = bloop.condition.Condition()
    other = User.email == "foo"

    assert (base & other) is other
    assert (base | other) is other
    assert (~base) is base
    assert len(base) == 0

    assert base.render(None) is None

    renderer.render(base, "condition")
    assert not renderer.rendered

def test_render_path(renderer, User):
    """ A path should be rendered as #column.#field.#field """
    path = "foo bar baz".split()
    renderer.name_ref(User.email, path=path)
    expected = {'ExpressionAttributeNames':
                {'#n0': 'email', '#n3': 'baz', '#n2': 'bar', '#n1': 'foo'}}
    assert renderer.rendered == expected

def test_path_comparitor(renderer, Document):
    """ Render paths for operations, comparisons, and multi-conditions """

    rating = Document.data["Rating"] > 0.5
    no_body = Document.data["Description"]["Body"].is_(None)
    stock = Document.data["Stock"].in_([1, 2, 3])
    condition = (rating & no_body) | stock

    expected = {
        'ConditionExpression': (
            '(((#n0.#n1 > :v2) AND (attribute_not_exists(#n0.#n3.#n4))) '
            'OR (#n0.#n5 IN (:v6, :v7, :v8)))'),
        'ExpressionAttributeValues': {
            ':v8': {'N': '3'}, ':v7': {'N': '2'},
            ':v6': {'N': '1'}, ':v2': {'N': '0.5'}},
        'ExpressionAttributeNames': {
            '#n0': 'data', '#n3': 'Description',
            '#n5': 'Stock', '#n1': 'Rating', '#n4': 'Body'}}

    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_typedmap_path_comparitor(renderer, engine, base_model, local_bind):
    """ TypedMap should defer to the value typedef for conditions """
    class Model(base_model):
        id = bloop.Column(bloop.Integer, hash_key=True)
        data = bloop.Column(bloop.TypedMap(bloop.UUID))
    with local_bind():
        engine.bind(base=base_model)

    uid = uuid.uuid4()
    condition = Model.data['foo'].is_(uid)

    expected = {
        'ConditionExpression': '(#n0.#n1 = :v2)',
        'ExpressionAttributeValues': {':v2': {'S': str(uid)}},
        'ExpressionAttributeNames': {'#n0': 'data', '#n1': 'foo'}
    }

    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_name_ref_with_path(
        renderer, engine, base_model, local_bind, document_type):
    """ Columns with custom names with literal periods render correctly """

    class Model(base_model):
        id = bloop.Column(bloop.Integer, hash_key=True, name='this.is.id')
        data = bloop.Column(document_type)
    with local_bind():
        engine.bind(base=base_model)

    no_id = Model.id.is_(None)
    path_condition = Model.data["Rating"] >= 2
    condition = no_id & path_condition

    expected = {
        'ExpressionAttributeNames': {
            '#n0': 'this.is.id', '#n2': 'Rating', '#n1': 'data'},
        'ExpressionAttributeValues': {':v3': {'N': '2'}},
        'ConditionExpression':
            '((attribute_not_exists(#n0)) AND (#n1.#n2 >= :v3))'}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_list_path(renderer, Document):
    """ render list indexes correctly """
    condition = Document.numbers[1] >= 3
    expected = {
        'ExpressionAttributeValues': {':v1': {'N': '3'}},
        'ConditionExpression': '(#n0[1] >= :v1)',
        'ExpressionAttributeNames': {'#n0': 'numbers'}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_equality(Document):
    lt = Document.id < 10
    gt = Document.id > 12

    path = Document.data["Rating"] == 3.4

    # Order doesn't matter for multi conditions
    basic_and = lt & gt
    swapped_and = gt & lt
    multiple_and = lt & lt & gt

    basic_or = lt | gt
    swapped_or = gt | lt
    multiple_or = lt | lt | gt

    not_lt = ~lt
    not_gt = ~gt

    not_exists_data = Document.data.is_(None)
    not_exists_id = Document.id.is_(None)
    exists_id = Document.id.is_not(None)

    begins_hello = Document.data["Description"]["Body"].begins_with("hello")
    begins_world = Document.data["Description"]["Body"].begins_with("world")
    begins_numbers = Document.numbers.begins_with(8)

    contains_hello = Document.data["Description"]["Body"].contains("hello")
    contains_world = Document.data["Description"]["Body"].contains("world")
    contains_numbers = Document.numbers.contains(9)

    between_small = Document.id.between(5, 6)
    between_big = Document.id.between(100, 200)
    between_numbers = Document.numbers.between(set([8080]), set([8088]))

    in_small = Document.id.in_([3, 7, 11])
    in_big = Document.id.in_([123, 456])
    in_numbers = Document.numbers.in_([120, 450])

    all_conditions = [
        lt, gt, path,
        basic_and, swapped_and, multiple_and,
        basic_or, swapped_or, multiple_or,
        not_lt, not_gt,
        not_exists_data, not_exists_id, exists_id,
        begins_hello, begins_world, begins_numbers,
        contains_hello, contains_world, contains_numbers,
        between_small, between_big, between_numbers,
        in_small, in_big, in_numbers
    ]

    for first in all_conditions:
        for second in all_conditions:
            if first is second:
                assert first == second
            else:
                assert first != second
<commit_msg>Use equality checks for Conditions in tests #34<commit_after>import bloop
import bloop.condition
import pytest
import uuid

from bloop.condition import And, Or, Not

def test_duplicate_name_refs(renderer, User):
    """ name refs are re-used for the same name """
    assert renderer.name_ref(User.age) == renderer.name_ref(User.age) == "#n0"

def test_no_refs(renderer):
    """
    when name/value refs are missing, ExpressionAttributeNames/Values
    aren't populated """
    condition = And()
    expected = {"ConditionExpression": "()"}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_condition_ops(User):
    age, name = (User.age >= 3), (User.name == "foo")

    assert age & name == And(age, name)
    assert age | name == Or(age, name)
    assert ~age == Not(age)

def test_condition_len(User):
    age, name = (User.age >= 3), (User.name == "foo")
    and_condition = age & name
    or_condition = bloop.condition.And(age, name, age)
    not_condition = ~age

    assert len(or_condition) == 3
    assert len(and_condition) == 2
    assert len(age) == len(name) == len(not_condition) == 1

def test_multi_shortcut(User):
    """ And or Or with single conditions render as their sole condition """
    age = User.age >= 3
    condition = bloop.condition.And(age)
    assert condition.conditions == [age]

    condition = bloop.condition.Or(age)
    assert condition.conditions == [age]

def test_multi_chains_flatten(User):
    """
    ((condition & condition) & condition) flattens the AND into one condition
    """
    age = User.age >= 3
    name = User.name == "foo"
    email = User.email != "bar"

    and_condition = bloop.condition.Condition()
    or_condition = bloop.condition.Condition()
    for c in [age, name, email]:
        and_condition &= c
        or_condition |= c
    assert and_condition == bloop.condition.And(age, name, email)
    assert or_condition == bloop.condition.Or(age, name, email)

def test_not(renderer, User):
    age = ~(User.age >= 3)
    condition = bloop.condition.And(age)
    expected = {"ConditionExpression": "(NOT (#n0 >= :v1))",
                "ExpressionAttributeNames": {"#n0": "age"},
                "ExpressionAttributeValues": {":v1": {"N": "3"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_invalid_comparator(User):
    with pytest.raises(ValueError):
        bloop.condition.Comparison(User.age, "not-a-comparator", 5)

def test_attribute_exists(User, renderer):
    exists = User.age.is_not(None)
    expected_exists = {"ConditionExpression": "(attribute_exists(#n0))",
                       "ExpressionAttributeNames": {"#n0": "age"}}

    renderer.render(exists, "condition")
    assert renderer.rendered == expected_exists

def test_attribute_not_exists(User, renderer):
    not_exists = User.age.is_(None)
    expected_not_exists = {
        "ConditionExpression": "(attribute_not_exists(#n0))",
        "ExpressionAttributeNames": {"#n0": "age"}}

    renderer.render(not_exists, "condition")
    assert renderer.rendered == expected_not_exists

def test_begins_with(renderer, User):
    condition = User.name.begins_with("foo")
    expected = {"ConditionExpression": "(begins_with(#n0, :v1))",
                "ExpressionAttributeNames": {"#n0": "name"},
                "ExpressionAttributeValues": {":v1": {"S": "foo"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_contains(renderer, User):
    condition = User.name.contains("foo")
    expected = {"ConditionExpression": "(contains(#n0, :v1))",
                "ExpressionAttributeNames": {"#n0": "name"},
                "ExpressionAttributeValues": {":v1": {"S": "foo"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_between(renderer, User):
    condition = User.name.between("bar", "foo")
    expected = {"ConditionExpression": "(#n0 BETWEEN :v1 AND :v2)",
                "ExpressionAttributeNames": {"#n0": "name"},
                "ExpressionAttributeValues": {":v1": {"S": "bar"},
                                              ":v2": {"S": "foo"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_in(renderer, User):
    condition = User.name.in_(["bar", "foo"])
    expected = {"ConditionExpression": "(#n0 IN (:v1, :v2))",
                "ExpressionAttributeNames": {"#n0": "name"},
                "ExpressionAttributeValues": {":v1": {"S": "bar"},
                                              ":v2": {"S": "foo"}}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_base_condition(renderer, User):
    """ (Condition() OP condition) is condition """
    base = bloop.condition.Condition()
    other = User.email == "foo"

    assert (base & other) is other
    assert (base | other) is other
    assert (~base) is base
    assert len(base) == 0

    assert base.render(None) is None

    renderer.render(base, "condition")
    assert not renderer.rendered

def test_render_path(renderer, User):
    """ A path should be rendered as #column.#field.#field """
    path = "foo bar baz".split()
    renderer.name_ref(User.email, path=path)
    expected = {'ExpressionAttributeNames':
                {'#n0': 'email', '#n3': 'baz', '#n2': 'bar', '#n1': 'foo'}}
    assert renderer.rendered == expected

def test_path_comparator(renderer, Document):
    """ Render paths for operations, comparisons, and multi-conditions """

    rating = Document.data["Rating"] > 0.5
    no_body = Document.data["Description"]["Body"].is_(None)
    stock = Document.data["Stock"].in_([1, 2, 3])
    condition = (rating & no_body) | stock

    expected = {
        'ConditionExpression': (
            '(((#n0.#n1 > :v2) AND (attribute_not_exists(#n0.#n3.#n4))) '
            'OR (#n0.#n5 IN (:v6, :v7, :v8)))'),
        'ExpressionAttributeValues': {
            ':v8': {'N': '3'}, ':v7': {'N': '2'},
            ':v6': {'N': '1'}, ':v2': {'N': '0.5'}},
        'ExpressionAttributeNames': {
            '#n0': 'data', '#n3': 'Description',
            '#n5': 'Stock', '#n1': 'Rating', '#n4': 'Body'}}

    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_typedmap_path_comparator(renderer, engine, base_model, local_bind):
    """ TypedMap should defer to the value typedef for conditions """
    class Model(base_model):
        id = bloop.Column(bloop.Integer, hash_key=True)
        data = bloop.Column(bloop.TypedMap(bloop.UUID))
    with local_bind():
        engine.bind(base=base_model)

    uid = uuid.uuid4()
    condition = Model.data['foo'].is_(uid)

    expected = {
        'ConditionExpression': '(#n0.#n1 = :v2)',
        'ExpressionAttributeValues': {':v2': {'S': str(uid)}},
        'ExpressionAttributeNames': {'#n0': 'data', '#n1': 'foo'}
    }

    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_name_ref_with_path(
        renderer, engine, base_model, local_bind, document_type):
    """ Columns with custom names with literal periods render correctly """

    class Model(base_model):
        id = bloop.Column(bloop.Integer, hash_key=True, name='this.is.id')
        data = bloop.Column(document_type)
    with local_bind():
        engine.bind(base=base_model)

    no_id = Model.id.is_(None)
    path_condition = Model.data["Rating"] >= 2
    condition = no_id & path_condition

    expected = {
        'ExpressionAttributeNames': {
            '#n0': 'this.is.id', '#n2': 'Rating', '#n1': 'data'},
        'ExpressionAttributeValues': {':v3': {'N': '2'}},
        'ConditionExpression':
            '((attribute_not_exists(#n0)) AND (#n1.#n2 >= :v3))'}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_list_path(renderer, Document):
    """ render list indexes correctly """
    condition = Document.numbers[1] >= 3
    expected = {
        'ExpressionAttributeValues': {':v1': {'N': '3'}},
        'ConditionExpression': '(#n0[1] >= :v1)',
        'ExpressionAttributeNames': {'#n0': 'numbers'}}
    renderer.render(condition, "condition")
    assert renderer.rendered == expected

def test_equality(Document):
    lt = Document.id < 10
    gt = Document.id > 12

    path = Document.data["Rating"] == 3.4

    # Order doesn't matter for multi conditions
    basic_and = lt & gt
    swapped_and = gt & lt
    multiple_and = lt & lt & gt

    basic_or = lt | gt
    swapped_or = gt | lt
    multiple_or = lt | lt | gt

    not_lt = ~lt
    not_gt = ~gt

    not_exists_data = Document.data.is_(None)
    not_exists_id = Document.id.is_(None)
    exists_id = Document.id.is_not(None)

    begins_hello = Document.data["Description"]["Body"].begins_with("hello")
    begins_world = Document.data["Description"]["Body"].begins_with("world")
    begins_numbers = Document.numbers.begins_with(8)

    contains_hello = Document.data["Description"]["Body"].contains("hello")
    contains_world = Document.data["Description"]["Body"].contains("world")
    contains_numbers = Document.numbers.contains(9)

    between_small = Document.id.between(5, 6)
    between_big = Document.id.between(100, 200)
    between_numbers = Document.numbers.between(set([8080]), set([8088]))

    in_small = Document.id.in_([3, 7, 11])
    in_big = Document.id.in_([123, 456])
    in_numbers = Document.numbers.in_([120, 450])

    all_conditions = [
        lt, gt, path,
        basic_and, swapped_and, multiple_and,
        basic_or, swapped_or, multiple_or,
        not_lt, not_gt,
        not_exists_data, not_exists_id, exists_id,
        begins_hello, begins_world, begins_numbers,
        contains_hello, contains_world, contains_numbers,
        between_small, between_big, between_numbers,
        in_small, in_big, in_numbers
    ]

    for first in all_conditions:
        for second in all_conditions:
            if first is second:
                assert first == second
            else:
                assert first != second
<|endoftext|>


<commit_before>"""
django-parler uses caching to avoid fetching model data when it doesn't have to.

These functions are used internally by django-parler to fetch model data.
Since all calls to the translation table are routed through our model descriptor fields,
cache access and expiry is rather simple to implement.
"""
from django.core.cache import cache
from django.utils import six
from parler import appsettings
from parler.utils import get_language_settings

if six.PY3:
    long = int

try:
    DEFAULT_TIMEOUT = cache.cache.default_timeout
except AttributeError:
    DEFAULT_TIMEOUT = 0

class IsMissing(object):
    # Allow _get_any_translated_model() to evaluate this as False.
    def __nonzero__(self):
        return False   # Python 2
    def __bool__(self):
        return False   # Python 3
    def __repr__(self):
        return "<IsMissing>"

MISSING = IsMissing()  # sentinel value

def get_object_cache_keys(instance):
    """
    Return the cache keys associated with an object.
    """
    if instance.pk is None or instance._state.adding:
        return []

    keys = []
    tr_models = instance._parler_meta.get_all_models()

    # TODO: performs a query to fetch the language codes. Store that in memcached too.
    for language in instance.get_available_languages():
        for tr_model in tr_models:
            keys.append(get_translation_cache_key(tr_model, instance.pk, language))

    return keys

def get_translation_cache_key(translated_model, master_id, language_code):
    """
    The low-level function to get the cache key for a translation.
    """
    # Always cache the entire object, as this already produces
    # a lot of queries. Don't go for caching individual fields.
    return 'parler.{0}.{1}.{2}.{3}'.format(translated_model._meta.app_label, translated_model.__name__, master_id, language_code)

def get_cached_translation(instance, language_code=None, related_name=None, use_fallback=False):
    """
    Fetch an cached translation.

    .. versionadded 1.2 Added the ``related_name`` parameter.
    """
    if language_code is None:
        language_code = instance.get_current_language()

    translated_model = instance._parler_meta.get_model_by_related_name(related_name)
    values = _get_cached_values(instance, translated_model, language_code, use_fallback)
    if not values:
        return None

    try:
        translation = translated_model(**values)
    except TypeError:
        # Some model field was removed, cache entry is no longer working.
        return None

    translation._state.adding = False
    return translation

def get_cached_translated_field(instance, field_name, language_code=None, use_fallback=False):
    """
    Fetch an cached field.
    """
    if language_code is None:
        language_code = instance.get_current_language()

    # In django-parler 1.1 the order of the arguments was fixed, It used to be language_code, field_name
    # This serves as detection against backwards incompatibility issues.
    if len(field_name) <= 5 and len(language_code) > 5:
        raise RuntimeError("Unexpected language code, did you swap field_name, language_code?")

    translated_model = instance._parler_meta.get_model_by_field(field_name)
    values = _get_cached_values(instance, translated_model, language_code, use_fallback)
    if not values:
        return None

    # Allow older cached versions where the field didn't exist yet.
    return values.get(field_name, None)

def _get_cached_values(instance, translated_model, language_code, use_fallback=False):
    """
    Fetch an cached field.
    """
    if not appsettings.PARLER_ENABLE_CACHING or not instance.pk or instance._state.adding:
        return None

    key = get_translation_cache_key(translated_model, instance.pk, language_code)
    values = cache.get(key)
    if not values:
        return None

    # Check for a stored fallback marker
    if values.get('__FALLBACK__', False):
        # Internal trick, already set the fallback marker, so no query will be performed.
        instance._translations_cache[translated_model][language_code] = MISSING

        # Allow to return the fallback language instead.
        if use_fallback:
            lang_dict = get_language_settings(language_code)
            # iterate over list of fallback languages, which should be already
            # in proper order
            for i_lang in lang_dict['fallbacks']:
                if i_lang != language_code:
                    return _get_cached_values(instance, translated_model, i_lang, use_fallback=False)
        return None

    values['master'] = instance
    values['language_code'] = language_code
    return values

def _cache_translation(translation, timeout=DEFAULT_TIMEOUT):
    """
    Store a new translation in the cache.
    """
    if not appsettings.PARLER_ENABLE_CACHING:
        return

    if translation.master_id is None:
        raise ValueError("Can't cache unsaved translation")

    # Cache a translation object.
    # For internal usage, object parameters are not suited for outside usage.
    fields = translation.get_translated_fields()
    values = {'id': translation.id}
    for name in fields:
        values[name] = getattr(translation, name)

    key = get_translation_cache_key(translation.__class__, translation.master_id, translation.language_code)
    cache.set(key, values, timeout=timeout)

def _cache_translation_needs_fallback(instance, language_code, related_name, timeout=DEFAULT_TIMEOUT):
    """
    Store the fact that a translation doesn't exist, and the fallback should be used.
    """
    if not appsettings.PARLER_ENABLE_CACHING or not instance.pk or instance._state.adding:
        return

    tr_model = instance._parler_meta.get_model_by_related_name(related_name)
    key = get_translation_cache_key(tr_model, instance.pk, language_code)
    cache.set(key, {'__FALLBACK__': True}, timeout=timeout)

def _delete_cached_translations(shared_model):
    cache.delete_many(get_object_cache_keys(shared_model))

def _delete_cached_translation(translation):
    if not appsettings.PARLER_ENABLE_CACHING:
        return

    # Delete a cached translation
    # For internal usage, object parameters are not suited for outside usage.
    key = get_translation_cache_key(translation.__class__, translation.master_id, translation.language_code)
    cache.delete(key)
<commit_msg>cache: fix pep8 and rename vars<commit_after>"""
django-parler uses caching to avoid fetching model data when it doesn't have to.

These functions are used internally by django-parler to fetch model data.
Since all calls to the translation table are routed through our model descriptor fields,
cache access and expiry is rather simple to implement.
"""
from django.core.cache import cache
from django.utils import six
from parler import appsettings
from parler.utils import get_language_settings

if six.PY3:
    long = int

try:
    DEFAULT_TIMEOUT = cache.cache.default_timeout
except AttributeError:
    DEFAULT_TIMEOUT = 0

class IsMissing(object):
    # Allow _get_any_translated_model() to evaluate this as False.
    def __nonzero__(self):
        return False   # Python 2
    def __bool__(self):
        return False   # Python 3
    def __repr__(self):
        return "<IsMissing>"

MISSING = IsMissing()  # sentinel value

def get_object_cache_keys(instance):
    """
    Return the cache keys associated with an object.
    """
    if instance.pk is None or instance._state.adding:
        return []

    keys = []
    tr_models = instance._parler_meta.get_all_models()

    # TODO: performs a query to fetch the language codes. Store that in memcached too.
    for language in instance.get_available_languages():
        for tr_model in tr_models:
            keys.append(get_translation_cache_key(tr_model, instance.pk, language))

    return keys

def get_translation_cache_key(translated_model, master_id, language_code):
    """
    The low-level function to get the cache key for a translation.
    """
    # Always cache the entire object, as this already produces
    # a lot of queries. Don't go for caching individual fields.
    return 'parler.{0}.{1}.{2}.{3}'.format(translated_model._meta.app_label, translated_model.__name__, master_id, language_code)

def get_cached_translation(instance, language_code=None, related_name=None, use_fallback=False):
    """
    Fetch an cached translation.

    .. versionadded 1.2 Added the ``related_name`` parameter.
    """
    if language_code is None:
        language_code = instance.get_current_language()

    translated_model = instance._parler_meta.get_model_by_related_name(related_name)
    values = _get_cached_values(instance, translated_model, language_code, use_fallback)
    if not values:
        return None

    try:
        translation = translated_model(**values)
    except TypeError:
        # Some model field was removed, cache entry is no longer working.
        return None

    translation._state.adding = False
    return translation

def get_cached_translated_field(instance, field_name, language_code=None, use_fallback=False):
    """
    Fetch an cached field.
    """
    if language_code is None:
        language_code = instance.get_current_language()

    # In django-parler 1.1 the order of the arguments was fixed, It used to be language_code, field_name
    # This serves as detection against backwards incompatibility issues.
    if len(field_name) <= 5 and len(language_code) > 5:
        raise RuntimeError("Unexpected language code, did you swap field_name, language_code?")

    translated_model = instance._parler_meta.get_model_by_field(field_name)
    values = _get_cached_values(instance, translated_model, language_code, use_fallback)
    if not values:
        return None

    # Allow older cached versions where the field didn't exist yet.
    return values.get(field_name, None)

def _get_cached_values(instance, translated_model, language_code, use_fallback=False):
    """
    Fetch an cached field.
    """
    if not appsettings.PARLER_ENABLE_CACHING or not instance.pk or instance._state.adding:
        return None

    key = get_translation_cache_key(translated_model, instance.pk, language_code)
    values = cache.get(key)
    if not values:
        return None

    # Check for a stored fallback marker
    if values.get('__FALLBACK__', False):
        # Internal trick, already set the fallback marker, so no query will be performed.
        instance._translations_cache[translated_model][language_code] = MISSING

        # Allow to return the fallback language instead.
        if use_fallback:
            lang_dict = get_language_settings(language_code)
            # iterate over list of fallback languages, which should be already
            # in proper order
            for fallback_lang in lang_dict['fallbacks']:
                if fallback_lang != language_code:
                    return _get_cached_values(
                        instance, translated_model, fallback_lang,
                        use_fallback=False
                    )
        return None

    values['master'] = instance
    values['language_code'] = language_code
    return values

def _cache_translation(translation, timeout=DEFAULT_TIMEOUT):
    """
    Store a new translation in the cache.
    """
    if not appsettings.PARLER_ENABLE_CACHING:
        return

    if translation.master_id is None:
        raise ValueError("Can't cache unsaved translation")

    # Cache a translation object.
    # For internal usage, object parameters are not suited for outside usage.
    fields = translation.get_translated_fields()
    values = {'id': translation.id}
    for name in fields:
        values[name] = getattr(translation, name)

    key = get_translation_cache_key(translation.__class__, translation.master_id, translation.language_code)
    cache.set(key, values, timeout=timeout)

def _cache_translation_needs_fallback(instance, language_code, related_name, timeout=DEFAULT_TIMEOUT):
    """
    Store the fact that a translation doesn't exist, and the fallback should be used.
    """
    if not appsettings.PARLER_ENABLE_CACHING or not instance.pk or instance._state.adding:
        return

    tr_model = instance._parler_meta.get_model_by_related_name(related_name)
    key = get_translation_cache_key(tr_model, instance.pk, language_code)
    cache.set(key, {'__FALLBACK__': True}, timeout=timeout)

def _delete_cached_translations(shared_model):
    cache.delete_many(get_object_cache_keys(shared_model))

def _delete_cached_translation(translation):
    if not appsettings.PARLER_ENABLE_CACHING:
        return

    # Delete a cached translation
    # For internal usage, object parameters are not suited for outside usage.
    key = get_translation_cache_key(translation.__class__, translation.master_id, translation.language_code)
    cache.delete(key)
<|endoftext|>


<commit_before>#!/usr/bin/python

import os.path
import re

def BuildFileAndTarget(build_file, target):
  # NOTE: If you just want to split up target into a build_file and target,
  # and you know that target already has a build_file that's been produced by
  # this function, pass '' for build_file.

  # NOTE: rsplit is used to disambiguate the Windows drive letter separator.
  target_split = target.rsplit(':', 1)
  if len(target_split) == 2:
    [build_file_rel, target] = target_split

    # If a relative path, build_file_rel is relative to the directory
    # containing build_file.  If build_file is not in the current directory,
    # build_file_rel is not a usable path as-is.  Resolve it by interpreting it
    # as relative to build_file.  If build_file_rel is absolute, it is usable
    # as a path regardless of the current directory, and os.path.join will
    # return it as-is.
    build_file = os.path.normpath(os.path.join(os.path.dirname(build_file),
                                               build_file_rel))

  return [build_file, target, build_file + ':' + target]

def QualifiedTarget(build_file, target):
  # "Qualified" means the file that a target was defined in and the target
  # name, separated by a colon.
  return BuildFileAndTarget(build_file, target)[2]

def RelativePath(path, relative_to):
  # Assuming both |path| and |relative_to| are relative to the current
  # directory, returns a relative path that identifies path relative to
  # relative_to.

  if os.path.isabs(path) != os.path.isabs(relative_to):
    # If one of the paths is absolute, both need to be absolute.
    path = os.path.abspath(path)
    relative_to = os.path.abspath(relative_to)
  else:
    # If both paths are relative, make sure they're normalized.
    path = os.path.normpath(path)
    relative_to = os.path.normpath(relative_to)

  # Split the paths into components.  As a special case, if either path is
  # the current directory, use an empty list as a split-up path.  This must be
  # done because the code that follows is unprepared to deal with "." meaning
  # "current directory" and it will instead assume that it's a subdirectory,
  # which is wrong.  It's possible to wind up with "." when it's passed to this
  # function, for example, by taking the dirname of a relative path in the
  # current directory.
  if path == os.path.curdir:
    path_split = []
  else:
    path_split = path.split(os.path.sep)

  if relative_to == os.path.curdir:
    relative_to_split = []
  else:
    relative_to_split = relative_to.split(os.path.sep)

  # Determine how much of the prefix the two paths share.
  prefix_len = len(os.path.commonprefix([path_split, relative_to_split]))

  # Put enough ".." components to back up out of relative_to to the common
  # prefix, and then append the part of path_split after the common prefix.
  relative_split = [os.path.pardir] * (len(relative_to_split) - prefix_len) + \
                   path_split[prefix_len:]

  # Turn it back into a string and we're done.
  return os.path.join(*relative_split)

# re objects used by EncodePOSIXShellArgument.  See IEEE 1003.1 XCU.2.2 at
# http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_02
# and the documentation for various shells.

# _quote is a pattern that should match any argument that needs to be quoted
# with double-quotes by EncodePOSIXShellArgument.  It matches the following
# characters appearing anywhere in an argument:
#   \t, \n, space  parameter separators
#   #              comments
#   $              expansions (quoted to always expand within one argument)
#   %              called out by IEEE 1003.1 XCU.2.2
#   &              job control
#   '              quoting
#   (, )           subshell execution
#   *, ?, [        pathname expansion
#   ;              command delimiter
#   <, >, |        redirection
#   =              assignment
#   {, }           brace expansion (bash)
#   ~              tilde expansion
# It also matches the empty string, because "" (or '') is the only way to
# represent an empty string literal argument to a POSIX shell.
#
# This does not match the characters in _escape, because those need to be
# backslash-escaped regardless of whether they appear in a double-quoted
# string.
_quote = re.compile('[\t\n #$%&\'()*;<=>?[{|}~]|^$')

# _escape is a pattern that should match any character that needs to be
# escaped with a backslash, whether or not the argument matched the _quote
# pattern.  _escape is used with re.sub to backslash anything in _escape's
# first match group, hence the (parentheses) in the regular expression.
#
# _escape matches the following characters appearing anywhere in an argument:
#   "  to prevent POSIX shells from interpreting this character for quoting
#   \  to prevent POSIX shells from interpreting this character for escaping
#   `  to prevent POSIX shells from interpreting this character for command
#      substitution
# Missing from this list is $, because the desired behavior of
# EncodePOSIXShellArgument is to permit parameter (variable) expansion.
#
# Also missing from this list is !, which bash will interpret as the history
# expansion character when history is enabled.  bash does not enable history
# by default in non-interactive shells, so this is not thought to be a problem.
# ! was omitted from this list because bash interprets "\!" as a literal string
# including the backslash character (avoiding history expansion but retaining
# the backslash), which would not be correct for argument encoding.  Handling
# this case properly would also be problematic because bash allows the history
# character to be changed with the histchars shell variable.  Fortunately,
# as history is not enabled in non-interactive shells and
# EncodePOSIXShellArgument is only expected to encode for non-interactive
# shells, there is no room for error here by ignoring !.
_escape = re.compile(r'(["\\`])')

def EncodePOSIXShellArgument(argument):
  """Encodes |argument| suitably for consumption by POSIX shells.

  argument may be quoted and escaped as necessary to ensure that POSIX shells
  treat the returned value as a literal representing the argument passed to
  this function.  Parameter (variable) expansions beginning with $ are allowed
  to remain intact without escaping the $, to allow the argument to contain
  references to variables to be expanded by the shell.
  """

  if _quote.search(argument):
    quote = '"'
  else:
    quote = ''

  encoded = quote + re.sub(_escape, r'\\\1', argument) + quote

  return encoded

def EncodePOSIXShellList(list):
  """Encodes |list| suitably for consumption by POSIX shells.

  Returns EncodePOSIXShellArgument for each item in list, and joins them
  together using the space character as an argument separator.
  """

  encoded_arguments = []
  for argument in list:
    encoded_arguments.append(EncodePOSIXShellArgument(argument))
  return ' '.join(encoded_arguments)
<commit_msg>Make RelativePath work when both path arguments are identical.<commit_after>#!/usr/bin/python

import os.path
import re

def BuildFileAndTarget(build_file, target):
  # NOTE: If you just want to split up target into a build_file and target,
  # and you know that target already has a build_file that's been produced by
  # this function, pass '' for build_file.

  # NOTE: rsplit is used to disambiguate the Windows drive letter separator.
  target_split = target.rsplit(':', 1)
  if len(target_split) == 2:
    [build_file_rel, target] = target_split

    # If a relative path, build_file_rel is relative to the directory
    # containing build_file.  If build_file is not in the current directory,
    # build_file_rel is not a usable path as-is.  Resolve it by interpreting it
    # as relative to build_file.  If build_file_rel is absolute, it is usable
    # as a path regardless of the current directory, and os.path.join will
    # return it as-is.
    build_file = os.path.normpath(os.path.join(os.path.dirname(build_file),
                                               build_file_rel))

  return [build_file, target, build_file + ':' + target]

def QualifiedTarget(build_file, target):
  # "Qualified" means the file that a target was defined in and the target
  # name, separated by a colon.
  return BuildFileAndTarget(build_file, target)[2]

def RelativePath(path, relative_to):
  # Assuming both |path| and |relative_to| are relative to the current
  # directory, returns a relative path that identifies path relative to
  # relative_to.

  if os.path.isabs(path) != os.path.isabs(relative_to):
    # If one of the paths is absolute, both need to be absolute.
    path = os.path.abspath(path)
    relative_to = os.path.abspath(relative_to)
  else:
    # If both paths are relative, make sure they're normalized.
    path = os.path.normpath(path)
    relative_to = os.path.normpath(relative_to)

  # Split the paths into components.  As a special case, if either path is
  # the current directory, use an empty list as a split-up path.  This must be
  # done because the code that follows is unprepared to deal with "." meaning
  # "current directory" and it will instead assume that it's a subdirectory,
  # which is wrong.  It's possible to wind up with "." when it's passed to this
  # function, for example, by taking the dirname of a relative path in the
  # current directory.
  if path == os.path.curdir:
    path_split = []
  else:
    path_split = path.split(os.path.sep)

  if relative_to == os.path.curdir:
    relative_to_split = []
  else:
    relative_to_split = relative_to.split(os.path.sep)

  # Determine how much of the prefix the two paths share.
  prefix_len = len(os.path.commonprefix([path_split, relative_to_split]))

  # Put enough ".." components to back up out of relative_to to the common
  # prefix, and then append the part of path_split after the common prefix.
  relative_split = [os.path.pardir] * (len(relative_to_split) - prefix_len) + \
                   path_split[prefix_len:]

  if len(relative_split) == 0:
    # The paths were the same.
    return ''

  # Turn it back into a string and we're done.
  return os.path.join(*relative_split)

# re objects used by EncodePOSIXShellArgument.  See IEEE 1003.1 XCU.2.2 at
# http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_02
# and the documentation for various shells.

# _quote is a pattern that should match any argument that needs to be quoted
# with double-quotes by EncodePOSIXShellArgument.  It matches the following
# characters appearing anywhere in an argument:
#   \t, \n, space  parameter separators
#   #              comments
#   $              expansions (quoted to always expand within one argument)
#   %              called out by IEEE 1003.1 XCU.2.2
#   &              job control
#   '              quoting
#   (, )           subshell execution
#   *, ?, [        pathname expansion
#   ;              command delimiter
#   <, >, |        redirection
#   =              assignment
#   {, }           brace expansion (bash)
#   ~              tilde expansion
# It also matches the empty string, because "" (or '') is the only way to
# represent an empty string literal argument to a POSIX shell.
#
# This does not match the characters in _escape, because those need to be
# backslash-escaped regardless of whether they appear in a double-quoted
# string.
_quote = re.compile('[\t\n #$%&\'()*;<=>?[{|}~]|^$')

# _escape is a pattern that should match any character that needs to be
# escaped with a backslash, whether or not the argument matched the _quote
# pattern.  _escape is used with re.sub to backslash anything in _escape's
# first match group, hence the (parentheses) in the regular expression.
#
# _escape matches the following characters appearing anywhere in an argument:
#   "  to prevent POSIX shells from interpreting this character for quoting
#   \  to prevent POSIX shells from interpreting this character for escaping
#   `  to prevent POSIX shells from interpreting this character for command
#      substitution
# Missing from this list is $, because the desired behavior of
# EncodePOSIXShellArgument is to permit parameter (variable) expansion.
#
# Also missing from this list is !, which bash will interpret as the history
# expansion character when history is enabled.  bash does not enable history
# by default in non-interactive shells, so this is not thought to be a problem.
# ! was omitted from this list because bash interprets "\!" as a literal string
# including the backslash character (avoiding history expansion but retaining
# the backslash), which would not be correct for argument encoding.  Handling
# this case properly would also be problematic because bash allows the history
# character to be changed with the histchars shell variable.  Fortunately,
# as history is not enabled in non-interactive shells and
# EncodePOSIXShellArgument is only expected to encode for non-interactive
# shells, there is no room for error here by ignoring !.
_escape = re.compile(r'(["\\`])')

def EncodePOSIXShellArgument(argument):
  """Encodes |argument| suitably for consumption by POSIX shells.

  argument may be quoted and escaped as necessary to ensure that POSIX shells
  treat the returned value as a literal representing the argument passed to
  this function.  Parameter (variable) expansions beginning with $ are allowed
  to remain intact without escaping the $, to allow the argument to contain
  references to variables to be expanded by the shell.
  """

  if _quote.search(argument):
    quote = '"'
  else:
    quote = ''

  encoded = quote + re.sub(_escape, r'\\\1', argument) + quote

  return encoded

def EncodePOSIXShellList(list):
  """Encodes |list| suitably for consumption by POSIX shells.

  Returns EncodePOSIXShellArgument for each item in list, and joins them
  together using the space character as an argument separator.
  """

  encoded_arguments = []
  for argument in list:
    encoded_arguments.append(EncodePOSIXShellArgument(argument))
  return ' '.join(encoded_arguments)
<|endoftext|>


<commit_before>from datetime import datetime
from django.contrib.auth.models import User
from django.contrib.sessions.models import Session
from django.db import models

import random

class UserProfile (models.Model):
    user = models.OneToOneField(User)
    avatar = models.ImageField(upload_to='static/game/image/avatars/', null=True, blank=True,
                               default='/static/game/image/avatars/default-avatar.jpeg')

class School (models.Model):
    name = models.CharField(max_length=200)

class Teacher (models.Model):
    name = models.CharField(max_length=200)
    user = models.OneToOneField(UserProfile)

class Class (models.Model):
    name = models.CharField(max_length=200)
    school = models.ForeignKey(School, related_name='class_school')
    teacher = models.ForeignKey(Teacher, related_name='class_teacher')

    def get_logged_in_students(self):
        """This gets all the students who are logged in."""
        sessions = Session.objects.filter(expire_date__gte=datetime.now())
        uid_list = []

        # Build a list of user ids from that query
        for session in sessions:
            data = session.get_decoded()
            uid_list.append(data.get('_auth_user_id', None))

        # Query all logged in users based on id list
        return Student.objects.filter(class_field=self).filter(user__id__in=uid_list)

    class Meta:
        verbose_name_plural = "classes"

class Student (models.Model):
    name = models.CharField(max_length=200)
    class_field = models.ForeignKey(Class, related_name='students')
    user = models.OneToOneField(UserProfile)

class Guardian (models.Model):
    name = models.CharField(max_length=200)
    children = models.ManyToManyField(Student)
    user = models.OneToOneField(UserProfile)

class Block (models.Model):
    type = models.CharField(max_length=200)

    def __unicode__(self):
        return self.type

class Level (models.Model):
    name = models.IntegerField()
    path = models.CharField(max_length=10000)
    default = models.BooleanField(default=False)
    owner = models.ForeignKey(UserProfile, related_name='levels', blank=True, null=True)
    blockLimit = models.IntegerField(blank=True, null=True)
    blocks = models.ManyToManyField(Block, related_name='+')

    @classmethod
    def random_road(cls):
        generated_path = Level.generate_random_path()
        level = cls(name=3000, path=generated_path)
        level.save()
        level.blocks = Block.objects.all()
        level.save()
        return level

    @staticmethod
    def generate_random_path():
        road_tiles = set()
        road_tiles.add((1,3))

        for _ in xrange(20):
            new_tile = Level.pick_adjacent_tile(road_tiles)
            if new_tile is not None:
                road_tiles.add(new_tile)

        print(road_tiles)
        return "[[0,3],[1,3],[1,4],[1,5],[1,6],[2,6],[2,5],[3,5],[3,4],[3,3],[3,2],[4,2],[4,3],[5,3]]"

    @staticmethod
    def pick_adjacent_tile(tiles):
        for attempts in xrange(5):
            origin = random.sample(tiles, 1)[0]
            possibles = set()
            if Level.is_possible((origin[0] - 1, origin[1]), tiles):
                possibles.add((origin[0] - 1, origin[1]))
            if Level.is_possible((origin[0] + 1, origin[1]), tiles):
                possibles.add((origin[0] + 1, origin[1]))
            if Level.is_possible((origin[0], origin[1] - 1), tiles):
                possibles.add((origin[0], origin[1] - 1))
            if Level.is_possible((origin[0], origin[1] + 1), tiles):
                possibles.add((origin[0], origin[1] + 1))

            if possibles:
                return random.sample(possibles, 1)[0]

        return None

    @staticmethod
    def is_possible(tile, tiles):
        return (tile not in tiles) and tile[0] > 0 and tile[0] < 10 - 1
                and tile[1] > 0 and tile[1] < 8 - 1

    def __unicode__(self):
        return 'Level ' + str(self.id)

class Attempt (models.Model):
    start_time = models.DateTimeField(auto_now_add=True)
    level = models.ForeignKey(Level, related_name='attempts')
    student = models.ForeignKey(Student, related_name='attempts')
    finish_time = models.DateTimeField(auto_now=True)
    score = models.FloatField()

class Command (models.Model):
    STEP_CHOICES = (
        ('Right', 'right'),
        ('Left', 'left'),
        ('Forward', 'forward'),
        ('TurnAround', 'turn around'),
        ('While', 'while'),
        ('If', 'if'),
    )

    step = models.IntegerField()
    attempt = models.ForeignKey(Attempt, related_name='commands')
    command = models.CharField(max_length=15, choices=STEP_CHOICES, default='Forward')
    next = models.IntegerField(blank=True, null=True)

    # Condition in While or If statements. Optional.
    condition = models.CharField(max_length=400, blank=True)
    # 'While' or 'If' block. Optional.
    executedBlock1 = models.CommaSeparatedIntegerField(blank=True, max_length=100)
    # 'Else' block. Optional.
    executedBlock2 = models.CommaSeparatedIntegerField(blank=True, max_length=100)
<commit_msg>Fix wrapped line<commit_after>from datetime import datetime
from django.contrib.auth.models import User
from django.contrib.sessions.models import Session
from django.db import models

import random

class UserProfile (models.Model):
    user = models.OneToOneField(User)
    avatar = models.ImageField(upload_to='static/game/image/avatars/', null=True, blank=True,
                               default='/static/game/image/avatars/default-avatar.jpeg')

class School (models.Model):
    name = models.CharField(max_length=200)

class Teacher (models.Model):
    name = models.CharField(max_length=200)
    user = models.OneToOneField(UserProfile)

class Class (models.Model):
    name = models.CharField(max_length=200)
    school = models.ForeignKey(School, related_name='class_school')
    teacher = models.ForeignKey(Teacher, related_name='class_teacher')

    def get_logged_in_students(self):
        """This gets all the students who are logged in."""
        sessions = Session.objects.filter(expire_date__gte=datetime.now())
        uid_list = []

        # Build a list of user ids from that query
        for session in sessions:
            data = session.get_decoded()
            uid_list.append(data.get('_auth_user_id', None))

        # Query all logged in users based on id list
        return Student.objects.filter(class_field=self).filter(user__id__in=uid_list)

    class Meta:
        verbose_name_plural = "classes"

class Student (models.Model):
    name = models.CharField(max_length=200)
    class_field = models.ForeignKey(Class, related_name='students')
    user = models.OneToOneField(UserProfile)

class Guardian (models.Model):
    name = models.CharField(max_length=200)
    children = models.ManyToManyField(Student)
    user = models.OneToOneField(UserProfile)

class Block (models.Model):
    type = models.CharField(max_length=200)

    def __unicode__(self):
        return self.type

class Level (models.Model):
    name = models.IntegerField()
    path = models.CharField(max_length=10000)
    default = models.BooleanField(default=False)
    owner = models.ForeignKey(UserProfile, related_name='levels', blank=True, null=True)
    blockLimit = models.IntegerField(blank=True, null=True)
    blocks = models.ManyToManyField(Block, related_name='+')

    @classmethod
    def random_road(cls):
        generated_path = Level.generate_random_path()
        level = cls(name=3000, path=generated_path)
        level.save()
        level.blocks = Block.objects.all()
        level.save()
        return level

    @staticmethod
    def generate_random_path():
        road_tiles = set()
        road_tiles.add((1,3))

        for _ in xrange(20):
            new_tile = Level.pick_adjacent_tile(road_tiles)
            if new_tile is not None:
                road_tiles.add(new_tile)

        print(road_tiles)
        return "[[0,3],[1,3],[1,4],[1,5],[1,6],[2,6],[2,5],[3,5],[3,4],[3,3],[3,2],[4,2],[4,3],[5,3]]"

    @staticmethod
    def pick_adjacent_tile(tiles):
        for attempts in xrange(5):
            origin = random.sample(tiles, 1)[0]
            possibles = set()
            if Level.is_possible((origin[0] - 1, origin[1]), tiles):
                possibles.add((origin[0] - 1, origin[1]))
            if Level.is_possible((origin[0] + 1, origin[1]), tiles):
                possibles.add((origin[0] + 1, origin[1]))
            if Level.is_possible((origin[0], origin[1] - 1), tiles):
                possibles.add((origin[0], origin[1] - 1))
            if Level.is_possible((origin[0], origin[1] + 1), tiles):
                possibles.add((origin[0], origin[1] + 1))

            if possibles:
                return random.sample(possibles, 1)[0]

        return None

    @staticmethod
    def is_possible(tile, tiles):
        return (tile not in tiles) and tile[0] > 0 and tile[0] < 10 - 1 and tile[1] > 0 and tile[1] < 8 - 1

    def __unicode__(self):
        return 'Level ' + str(self.id)

class Attempt (models.Model):
    start_time = models.DateTimeField(auto_now_add=True)
    level = models.ForeignKey(Level, related_name='attempts')
    student = models.ForeignKey(Student, related_name='attempts')
    finish_time = models.DateTimeField(auto_now=True)
    score = models.FloatField()

class Command (models.Model):
    STEP_CHOICES = (
        ('Right', 'right'),
        ('Left', 'left'),
        ('Forward', 'forward'),
        ('TurnAround', 'turn around'),
        ('While', 'while'),
        ('If', 'if'),
    )

    step = models.IntegerField()
    attempt = models.ForeignKey(Attempt, related_name='commands')
    command = models.CharField(max_length=15, choices=STEP_CHOICES, default='Forward')
    next = models.IntegerField(blank=True, null=True)

    # Condition in While or If statements. Optional.
    condition = models.CharField(max_length=400, blank=True)
    # 'While' or 'If' block. Optional.
    executedBlock1 = models.CommaSeparatedIntegerField(blank=True, max_length=100)
    # 'Else' block. Optional.
    executedBlock2 = models.CommaSeparatedIntegerField(blank=True, max_length=100)
<|endoftext|>


<commit_before>import os

def find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True):
    """
    Function that finds files in a directory based on substring matching.

    Parameters
    ----------

    substring : `str`
      Substring of the file to be matched.

    path : `str`
      Path where to look.

    recursive: `bool`
      If true, searches subdirectories recursively.

    check_ext: `str`
      If string (e.g., '.txt'), only returns files that
        match the specified file extension.

    ignore_invisible : `bool`
      If `True`, ignores invisible files (i.e., files starting with a period).

    Returns
    ----------
    results : `list`
      List of the matched files.

    """
    def check_file(f, path):
        if substring in f:
            compl_path = os.path.join(path, f)
            if os.path.isfile(compl_path):
                return compl_path
        return False

    results = []

    if recursive:
        for par, nxt, fnames in os.walk(path):
            for f in fnames:
                fn = check_file(f, par)
                if fn:
                    results.append(fn)

    else:
        for f in os.listdir(path):
            if invisible and f.startswith('.'):
                continue
            fn = check_file(f, path)
            if fn:
                results.append(fn)

    if check_ext:
        results = [r for r in results if os.path.splitext(r)[-1] == check_ext]

    return results

def find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True):
    """
    Function that finds and groups files from different directories in a python dictionary.

    Parameters
    ----------

    paths : `list`
      Paths of the directories to be searched. Dictionary keys are build from
        the first directory.

    substring : `str`
      Substring that all files have to contain to be considered.

    extensions : `list`
      `None` or `list` of allowed file extensions for each path. If provided, the number
        of extensions must match the number of `paths`.

    validity_check : `bool`
      If `True`, checks if all dictionary values have the same number of file paths. Prints
        a warning and returns an empty dictionary if the validity check failed.

    ignore_invisible : `bool`
      If `True`, ignores invisible files (i.e., files starting with a period).

    Returns
    ----------
    groups : `dict`
      Dictionary of files paths. Keys are the file names found in the first directory listed
        in `paths` (without file extension).

    """
    n = len(paths)

    # must have same number of paths and extensions
    assert(len(paths) >= 2)
    if extensions:
        assert(len(extensions) == n)
    else:
        extensions = ['' for i in range(n)]

    base = find_files(path=paths[0],  substring=substring, check_ext=extensions[0], ignore_invisible=ignore_invisible)
    rest = [find_files(path=paths[i],  substring=substring, check_ext=extensions[i], ignore_invisible=ignore_invisible) for i in range(1,n)]

    groups = {os.path.splitext(os.path.basename(f))[0]:[f] for f in base}

    for idx,r in enumerate(rest):
        for f in r:
            basename, ext = os.path.splitext(os.path.basename(f))
            try:
                if extensions[idx+1] == '' or ext == extensions[idx+1]:
                    groups[basename].append(f)
            except KeyError:
                pass

    if validity_check:
        num = sorted([(len(v),k) for k,v in groups.items()])
        for i in num[1:]:
            if i[0] < num[0][0]:
                print('Warning, key "%s" has less values than "key" %s.' % (i[1], num[0][1]))
                groups = {}

    return groups<commit_msg>ignore inv<commit_after>import os

def find_files(substring, path, recursive=False, check_ext=None, ignore_invisible=True):
    """
    Function that finds files in a directory based on substring matching.

    Parameters
    ----------

    substring : `str`
      Substring of the file to be matched.

    path : `str`
      Path where to look.

    recursive: `bool`
      If true, searches subdirectories recursively.

    check_ext: `str`
      If string (e.g., '.txt'), only returns files that
        match the specified file extension.

    ignore_invisible : `bool`
      If `True`, ignores invisible files (i.e., files starting with a period).

    Returns
    ----------
    results : `list`
      List of the matched files.

    """
    def check_file(f, path):
        if substring in f:
            compl_path = os.path.join(path, f)
            if os.path.isfile(compl_path):
                return compl_path
        return False

    results = []

    if recursive:
        for par, nxt, fnames in os.walk(path):
            for f in fnames:
                fn = check_file(f, par)
                if fn:
                    results.append(fn)

    else:
        for f in os.listdir(path):
            if ignore_invisible and f.startswith('.'):
                continue
            fn = check_file(f, path)
            if fn:
                results.append(fn)

    if check_ext:
        results = [r for r in results if os.path.splitext(r)[-1] == check_ext]

    return results

def find_filegroups(paths, substring='', extensions=None, validity_check=True, ignore_invisible=True):
    """
    Function that finds and groups files from different directories in a python dictionary.

    Parameters
    ----------

    paths : `list`
      Paths of the directories to be searched. Dictionary keys are build from
        the first directory.

    substring : `str`
      Substring that all files have to contain to be considered.

    extensions : `list`
      `None` or `list` of allowed file extensions for each path. If provided, the number
        of extensions must match the number of `paths`.

    validity_check : `bool`
      If `True`, checks if all dictionary values have the same number of file paths. Prints
        a warning and returns an empty dictionary if the validity check failed.

    ignore_invisible : `bool`
      If `True`, ignores invisible files (i.e., files starting with a period).

    Returns
    ----------
    groups : `dict`
      Dictionary of files paths. Keys are the file names found in the first directory listed
        in `paths` (without file extension).

    """
    n = len(paths)

    # must have same number of paths and extensions
    assert(len(paths) >= 2)
    if extensions:
        assert(len(extensions) == n)
    else:
        extensions = ['' for i in range(n)]

    base = find_files(path=paths[0],  substring=substring, check_ext=extensions[0], ignore_invisible=ignore_invisible)
    rest = [find_files(path=paths[i],  substring=substring, check_ext=extensions[i], ignore_invisible=ignore_invisible) for i in range(1,n)]

    groups = {os.path.splitext(os.path.basename(f))[0]:[f] for f in base}

    for idx,r in enumerate(rest):
        for f in r:
            basename, ext = os.path.splitext(os.path.basename(f))
            try:
                if extensions[idx+1] == '' or ext == extensions[idx+1]:
                    groups[basename].append(f)
            except KeyError:
                pass

    if validity_check:
        num = sorted([(len(v),k) for k,v in groups.items()])
        for i in num[1:]:
            if i[0] < num[0][0]:
                print('Warning, key "%s" has less values than "key" %s.' % (i[1], num[0][1]))
                groups = {}

    return groups<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import thriftpy
thriftpy.install_import_hook()

from thriftpy.utils import serialize, deserialize
from thriftpy.protocol import TCyBinaryProtocolFactory

import container_thrift as container

def test_list_item():
    l_item = container.ListItem()
    l_item.list_string = ['foo', 'bar']
    l_item.list_list_string = [['foo', 'bar']]

    b = serialize(l_item, TCyBinaryProtocolFactory())
    l_item2 = deserialize(container.ListItem(), b, TCyBinaryProtocolFactory())
    assert l_item == l_item2

def test_map_item():
    m_item = container.MapItem()
    m_item.map_string = {'foo': 'bar'}
    m_item.map_map_string = {'foo': {'hello': 'world'}}

    b = serialize(m_item, TCyBinaryProtocolFactory())
    m_item2 = deserialize(container.MapItem(), b, TCyBinaryProtocolFactory())
    assert m_item == m_item2

def test_mix_item():
    x_item = container.MixItem()
    x_item.list_map = [{'foo': 'bar'}]
    x_item.map_list = {'foo': ['hello', 'world']}

    b = serialize(x_item, TCyBinaryProtocolFactory())
    x_item2 = deserialize(container.MixItem(), b, TCyBinaryProtocolFactory())
    assert x_item == x_item2

def test_list_struct():
    l_item = container.ListItem()
    l_item.list_string = ['foo', 'bar'] * 100
    l_item.list_list_string = [['foo', 'bar']] * 100

    l_struct = container.ListStruct()
    l_struct.list_items = [l_item] * 100

    b = serialize(l_struct, TCyBinaryProtocolFactory())
    l_struct2 = deserialize(container.ListStruct(), b,
                            TCyBinaryProtocolFactory())
    assert l_struct == l_struct2
<commit_msg>revert to python binary in test_container.<commit_after># -*- coding: utf-8 -*-

import thriftpy
thriftpy.install_import_hook()

from thriftpy.utils import serialize, deserialize

import container_thrift as container

def test_list_item():
    l_item = container.ListItem()
    l_item.list_string = ['foo', 'bar']
    l_item.list_list_string = [['foo', 'bar']]

    b = serialize(l_item)
    l_item2 = deserialize(container.ListItem(), b)
    assert l_item == l_item2

def test_map_item():
    m_item = container.MapItem()
    m_item.map_string = {'foo': 'bar'}
    m_item.map_map_string = {'foo': {'hello': 'world'}}

    b = serialize(m_item)
    m_item2 = deserialize(container.MapItem(), b)
    assert m_item == m_item2

def test_mix_item():
    x_item = container.MixItem()
    x_item.list_map = [{'foo': 'bar'}]
    x_item.map_list = {'foo': ['hello', 'world']}

    b = serialize(x_item)
    x_item2 = deserialize(container.MixItem(), b)
    assert x_item == x_item2

def test_list_struct():
    l_item = container.ListItem()
    l_item.list_string = ['foo', 'bar'] * 100
    l_item.list_list_string = [['foo', 'bar']] * 100

    l_struct = container.ListStruct()
    l_struct.list_items = [l_item] * 100

    b = serialize(l_struct)
    l_struct2 = deserialize(container.ListStruct(), b)
    assert l_struct == l_struct2
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import logging
import re
import sys

import pyrapi

logging.basicConfig(level=logging.INFO)
_log = logging.getLogger('align')

def _read_fq_record(fp):
    # read 4 lines
    lines = list()
    lineno = 0
    for _ in xrange(4):
        lineno += 1
        line = fp.readline()
        if line:
            lines.append(line.rstrip('\n'))
        else:
            return None
    if lines[2] != '+':
        raise RuntimeError("Format error at line %d:\n%s" % (lineno, '\n'.join(lines)))

    retval = dict()
    retval['id'] = re.sub(r'^@\s*', '', lines[0])
    retval['seq'] = lines[1]
    retval['q'] = lines[3]
    return retval

def read_fastq_1(fp):
    done = False
    try:
        while not done:
            record1 = _read_fq_record(fp)
            record2 = _read_fq_record(fp)
            if record1 and record2:
                yield record1, record2
            else:
                done = True
    finally:
        fp.close()

def read_fastq_2(f1, f2):
    done = False
    try:
        while not done:
            record1 = _read_fq_record(f1)
            record2 = _read_fq_record(f2)
            if record1 and record2:
                yield record1, record2
            else:
                done = True
    finally:
        f1.close()
        f2.close()

def read_prq(fp):
    try:
        for line in fp:
            row = line.rstrip('\n').split('\t')
            record1 = dict( id=row[0], seq=row[1], q=row[2] )
            record2 = dict( id=row[0], seq=row[3], q=row[4] )
            yield record1, record2
    finally:
        fp.close()

def create_input(options):
    if options.format == 'fastq':
        if len(options.input) == 2:
            return read_fastq_2(options.input[0], options.input[1])
        elif len(options.input) == 1:
            return read_fastq_1(options.input[0])
        else:
            raise RuntimeError("Unexpected number of fastq input files %d!" % len(options.input))
    elif options.format == 'prq':
        if len(options.input) == 1:
            return read_prq(options.input[0])
        else:
            raise RuntimeError("Unexpected number of prq input files %d!" % len(options.input))

def parse_args(args=None):
    parser = argparse.ArgumentParser()
    parser.add_argument('ref')
    parser.add_argument('input', type=argparse.FileType('r'), nargs='*', default=sys.stdin)
    parser.add_argument('--format', choices=['fastq', 'prq'], default='fastq')
    parser.add_argument('--se', action="store_true", default=False)

    options = parser.parse_args(args)

    if len(options.input) > 2:
        parser.error("Number of input files cannot be greater than 2")
    elif len(options.input) == 0:
        raise RuntimeError("BUG! Empty options.input array")

    if options.se:
        raise NotImplementedError("Sorry. Single-end alignments not implemented")

    return options

def main(argv=None):
    options = parse_args(argv)
    plugin = pyrapi.load_aligner('rapi_bwa')

    opts = plugin.opts()
    plugin.init(opts)
    _log.info("Using the %s aligner plugin, aligner version %s, plugin version %s",
            plugin.aligner_name(), plugin.aligner_version(), plugin.plugin_version())
    pe = not options.se
    _log.info("Working in %s mode", 'paired-end' if pe else 'single-end')
    _log.info("Reading data in %s format", options.format)

    _log.info("Loading reference %s", options.ref)
    ref = plugin.ref(options.ref)
    _log.info("Reference loaded")

    batch = plugin.read_batch(2 if pe else 1)

    input_generator = create_input(options)

    _log.info('loading data')
    for reads in input_generator:
        read1 = reads[0]
        batch.append(read1['id'], read1['seq'], read1['q'], plugin.QENC_SANGER)
        if pe:
            read2 = reads[1]
            batch.append(read2['id'], read2['seq'], read2['q'], plugin.QENC_SANGER)

    _log.info('Loaded %s reads', len(batch))

    _log.info("aligning...")
    aligner = plugin.aligner(opts)
    aligner.align_reads(ref, batch)
    _log.info("finished aligning")

    _log.info("Here's the output")
    print plugin.format_sam_hdr(ref)
    for idx, fragment in enumerate(batch):
        _log.info('fragment')
        assert len(fragment) == 2
        sam = plugin.format_sam(batch, idx)
        print sam

    ref.unload()

if __name__ == '__main__':
    main()
<commit_msg>Work in batches<commit_after>#!/usr/bin/env python

import argparse
import logging
import re
import sys

import pyrapi

logging.basicConfig(level=logging.INFO)
_log = logging.getLogger('align')

def _read_fq_record(fp):
    # read 4 lines
    lines = list()
    lineno = 0
    for _ in xrange(4):
        lineno += 1
        line = fp.readline()
        if line:
            lines.append(line.rstrip('\n'))
        else:
            return None
    if lines[2] != '+':
        raise RuntimeError("Format error at line %d:\n%s" % (lineno, '\n'.join(lines)))

    retval = dict()
    retval['id'] = re.sub(r'^@\s*', '', lines[0])
    retval['seq'] = lines[1]
    retval['q'] = lines[3]
    return retval

def read_fastq_1(fp):
    done = False
    try:
        while not done:
            record1 = _read_fq_record(fp)
            record2 = _read_fq_record(fp)
            if record1 and record2:
                yield record1, record2
            else:
                done = True
    finally:
        fp.close()

def read_fastq_2(f1, f2):
    done = False
    try:
        while not done:
            record1 = _read_fq_record(f1)
            record2 = _read_fq_record(f2)
            if record1 and record2:
                yield record1, record2
            else:
                done = True
    finally:
        f1.close()
        f2.close()

def read_prq(fp):
    try:
        for line in fp:
            row = line.rstrip('\n').split('\t')
            record1 = dict( id=row[0], seq=row[1], q=row[2] )
            record2 = dict( id=row[0], seq=row[3], q=row[4] )
            yield record1, record2
    finally:
        fp.close()

def create_input(options):
    if options.format == 'fastq':
        if len(options.input) == 2:
            return read_fastq_2(options.input[0], options.input[1])
        elif len(options.input) == 1:
            return read_fastq_1(options.input[0])
        else:
            raise RuntimeError("Unexpected number of fastq input files %d!" % len(options.input))
    elif options.format == 'prq':
        if len(options.input) == 1:
            return read_prq(options.input[0])
        else:
            raise RuntimeError("Unexpected number of prq input files %d!" % len(options.input))

def parse_args(args=None):
    parser = argparse.ArgumentParser()
    parser.add_argument('ref')
    parser.add_argument('input', type=argparse.FileType('r'), nargs='*', default=sys.stdin)
    parser.add_argument('--format', choices=['fastq', 'prq'], default='fastq')
    parser.add_argument('--se', action="store_true", default=False)

    options = parser.parse_args(args)

    if len(options.input) > 2:
        parser.error("Number of input files cannot be greater than 2")
    elif len(options.input) == 0:
        raise RuntimeError("BUG! Empty options.input array")

    if options.se:
        raise NotImplementedError("Sorry. Single-end alignments not implemented")

    return options

def main(argv=None):
    batch_size = 100000 # n reads
    assert batch_size % 2 == 0
    options = parse_args(argv)
    plugin = pyrapi.load_aligner('rapi_bwa')

    opts = plugin.opts()
    plugin.init(opts)
    _log.info("Using the %s aligner plugin, aligner version %s, plugin version %s",
            plugin.aligner_name(), plugin.aligner_version(), plugin.plugin_version())
    pe = not options.se
    _log.info("Working in %s mode", 'paired-end' if pe else 'single-end')
    _log.info("Reading data in %s format", options.format)

    _log.info("Loading reference %s", options.ref)
    ref = plugin.ref(options.ref)
    _log.info("Reference loaded")

    batch = plugin.read_batch(2 if pe else 1)
    # allocate space for reads
    batch.reserve(100000 / batch.n_reads_per_frag)

    aligner = plugin.aligner(opts)

    # print SAM header
    print plugin.format_sam_hdr(ref)

    input_generator = create_input(options)

    def _load_batch():
        batch.clear()
        _log.info('loading batch %s', batch_count)
        for reads in input_generator:
            read1 = reads[0]
            batch.append(read1['id'], read1['seq'], read1['q'], plugin.QENC_SANGER)
            if pe:
                read2 = reads[1]
                batch.append(read2['id'], read2['seq'], read2['q'], plugin.QENC_SANGER)
            if len(batch) >= batch_size:
                break
        # return whether or not the batch is empty
        return len(batch) != 0

    def _process_batch():
        _log.info("aligning...")
        aligner.align_reads(ref, batch)
        _log.info("finished aligning. Printing output")
        for idx in xrange(batch.n_fragments):
            sam = plugin.format_sam(batch, idx)
            print sam

    done = False

    batch_count = 0
    while not done:
        batch_count += 1
        _log.info("Batch %s", batch_count)
        has_data = _load_batch()
        if has_data:
            _process_batch()
        else:
            done = True

    ref.unload()

if __name__ == '__main__':
    main()
<|endoftext|>