<commit_before>import numpy as np

def assert_mesh_consistency(mesh):
    assert np.all(np.logical_xor(mesh.is_boundary_facet, mesh.is_interior_facet))

    bpts = np.array(
        [
            mesh.is_boundary_point,
            mesh.is_interior_point,
            ~mesh.is_point_used,
        ]
    )
    assert np.all(np.sum(bpts, axis=0) == 1)

    # consistency check for facets_cells
    assert np.all(mesh.is_boundary_facet[mesh.facets_cells["boundary"][0]])
    assert not np.any(mesh.is_boundary_facet[mesh.facets_cells["interior"][0]])

    for edge_id, cell_id, local_edge_id in mesh.facets_cells["boundary"].T:
        assert edge_id == mesh.cells["facets"][cell_id][local_edge_id]

    for (
        edge_id,
        cell_id0,
        cell_id1,
        local_edge_id0,
        local_edge_id1,
    ) in mesh.facets_cells["interior"].T:
        assert edge_id == mesh.cells["facets"][cell_id0][local_edge_id0]
        assert edge_id == mesh.cells["facets"][cell_id1][local_edge_id1]

    # check consistency of facets_cells_idx with facets_cells
    for edge_id, idx in enumerate(mesh.facets_cells_idx):
        if mesh.is_boundary_facet[edge_id]:
            assert edge_id == mesh.facets_cells["boundary"][0, idx]
        else:
            assert edge_id == mesh.facets_cells["interior"][0, idx]

    # Assert facets_cells integrity
    for cell_gid, edge_gids in enumerate(mesh.cells["facets"]):
        for edge_gid in edge_gids:
            idx = mesh.facets_cells_idx[edge_gid]
            if mesh.is_boundary_facet[edge_gid]:
                assert cell_gid == mesh.facets_cells["boundary"][1, idx]
            else:
                assert cell_gid in mesh.facets_cells["interior"][1:3, idx]

    # make sure the edges are opposite of the points
    for cell_gid, (point_ids, edge_ids) in enumerate(
        zip(mesh.cells["points"], mesh.cells["facets"])
    ):
        for k in range(len(point_ids)):
            assert set(point_ids) == set(
                [*mesh.edges["points"][edge_ids][k], point_ids[k]]
            )

    # make sure the is_boundary_point/edge/cell is consistent
    ref_cells = np.any(mesh.is_boundary_facet_local, axis=0)
    print(mesh.points)
    print(mesh.cells)
    assert np.all(mesh.is_boundary_cell == ref_cells)
    ref_points = np.zeros(len(mesh.points), dtype=bool)
    ref_points[mesh.idx[1][:, mesh.is_boundary_facet_local]] = True
    assert np.all(mesh.is_boundary_point == ref_points)

    assert len(mesh.control_volumes) == len(mesh.points)
    assert len(mesh.control_volume_centroids) == len(mesh.points)

    # TODO add more consistency checks

def compute_all_entities(mesh):
    mesh.is_boundary_point
    mesh.is_interior_point
    mesh.is_boundary_facet_local
    mesh.is_boundary_facet
    mesh.is_boundary_cell
    mesh.cell_volumes
    mesh.ce_ratios
    mesh.signed_cell_volumes
    mesh.cell_centroids
    mesh.control_volumes
    mesh.create_facets()
    mesh.facets_cells
    mesh.facets_cells_idx
    mesh.boundary_facets
    mesh.interior_facets
    mesh.cell_circumcenters
    mesh.signed_circumcenter_distances
    mesh.control_volume_centroids

    assert mesh._cv_cell_mask is not None
    assert mesh.edges is not None
    assert mesh.subdomains is not {}
    assert mesh._is_interior_point is not None
    assert mesh._is_boundary_point is not None
    assert mesh._is_boundary_facet_local is not None
    assert mesh._is_boundary_facet is not None
    assert mesh._is_boundary_cell is not None
    assert mesh._facets_cells is not None
    assert mesh._facets_cells_idx is not None
    assert mesh._boundary_facets is not None
    assert mesh._interior_facets is not None
    assert mesh._is_point_used is not None
    assert mesh._half_edge_coords is not None
    assert mesh._ei_dot_ei is not None
    assert mesh._volumes is not None
    assert mesh._ce_ratios is not None
    assert mesh._circumcenters is not None
    assert mesh._signed_circumcenter_distances is not None
    assert mesh._control_volumes is not None
    assert mesh._cell_partitions is not None
    assert mesh._cv_centroids is not None
    assert mesh._cvc_cell_mask is not None
    assert mesh._signed_cell_volumes is not None
    assert mesh._cell_centroids is not None

def assert_mesh_equality(mesh0, mesh1):
    assert np.all(mesh0.cells["points"] == mesh1.cells["points"])
    assert np.all(np.abs(mesh0.points - mesh1.points) < 1.0e-14)

    assert np.all(np.abs(mesh0.points - mesh1.points) < 1.0e-14)
    assert np.all(mesh0.edges["points"] == mesh1.edges["points"])

    # # Assume that in mesh1, the rows are ordered such that the edge indices [0] are in
    # # order. The mesh0 array isn't order in many cases, e.g., if cells were removed and
    # # rows appened the boundary array. Hence, justsort the array in mesh0 and compare.
    # k = np.argsort(mesh0.facets_cells["boundary"][0])
    # assert np.all(mesh0.facets_cells["boundary"][:, k] == mesh1.facets_cells["boundary"])

    # # The interior facets_cells are already ordered, even after remove_cells(). (As
    # # opposed to boundary edges, there can be no new interior edges, just some are
    # # removed.)
    # print(mesh0.facets_cells["interior"].T)
    # print()
    # print(mesh1.facets_cells["interior"].T)
    # assert np.all(mesh0.facets_cells["interior"] == mesh1.facets_cells["interior"])
    # exit(1)
    # # print()
    # # print(mesh0.facets_cells["interior"])
    # # print()
    # # print(mesh1.facets_cells["interior"])
    #
    # # These should not be equal; see reordering above
    # assert np.all(mesh0.facets_cells_idx == mesh1.facets_cells_idx)

    assert np.all(mesh0.boundary_facets == mesh1.boundary_facets)
    assert np.all(mesh0.interior_facets == mesh1.interior_facets)

    assert np.all(mesh0.is_point_used == mesh1.is_point_used)
    assert np.all(mesh0.is_boundary_point == mesh1.is_boundary_point)
    assert np.all(mesh0.is_interior_point == mesh1.is_interior_point)
    assert np.all(mesh0.is_boundary_facet_local == mesh1.is_boundary_facet_local)
    assert np.all(mesh0.is_boundary_facet == mesh1.is_boundary_facet)
    assert np.all(mesh0.is_boundary_cell == mesh1.is_boundary_cell)

    assert np.all(np.abs(mesh0.ei_dot_ei - mesh1.ei_dot_ei) < 1.0e-14)
    assert np.all(np.abs(mesh0.cell_volumes - mesh1.cell_volumes) < 1.0e-14)
    assert np.all(np.abs(mesh0.ce_ratios - mesh1.ce_ratios) < 1.0e-14)
    assert np.all(
        np.abs(
            mesh0.signed_circumcenter_distances - mesh1.signed_circumcenter_distances
        )
        < 1.0e-14
    )
    assert np.all(
        np.abs(mesh0.signed_cell_volumes - mesh1.signed_cell_volumes) < 1.0e-14
    )
    assert np.all(np.abs(mesh0.cell_centroids - mesh1.cell_centroids) < 1.0e-14)
    assert np.all(np.abs(mesh0.cell_circumcenters - mesh1.cell_circumcenters) < 1.0e-14)
    assert np.all(np.abs(mesh0.control_volumes - mesh1.control_volumes) < 1.0e-14)

    ipu = mesh0.is_point_used
    assert np.all(
        np.abs(
            mesh0.control_volume_centroids[ipu] - mesh1.control_volume_centroids[ipu]
        )
        < 1.0e-14
    )
<commit_msg>tests: bit stricter mesh comparison<commit_after>import numpy as np

def assert_mesh_consistency(mesh):
    assert np.all(np.logical_xor(mesh.is_boundary_facet, mesh.is_interior_facet))

    bpts = np.array(
        [
            mesh.is_boundary_point,
            mesh.is_interior_point,
            ~mesh.is_point_used,
        ]
    )
    assert np.all(np.sum(bpts, axis=0) == 1)

    # consistency check for facets_cells
    assert np.all(mesh.is_boundary_facet[mesh.facets_cells["boundary"][0]])
    assert not np.any(mesh.is_boundary_facet[mesh.facets_cells["interior"][0]])

    for edge_id, cell_id, local_edge_id in mesh.facets_cells["boundary"].T:
        assert edge_id == mesh.cells["facets"][cell_id][local_edge_id]

    for (
        edge_id,
        cell_id0,
        cell_id1,
        local_edge_id0,
        local_edge_id1,
    ) in mesh.facets_cells["interior"].T:
        assert edge_id == mesh.cells["facets"][cell_id0][local_edge_id0]
        assert edge_id == mesh.cells["facets"][cell_id1][local_edge_id1]

    # check consistency of facets_cells_idx with facets_cells
    for edge_id, idx in enumerate(mesh.facets_cells_idx):
        if mesh.is_boundary_facet[edge_id]:
            assert edge_id == mesh.facets_cells["boundary"][0, idx]
        else:
            assert edge_id == mesh.facets_cells["interior"][0, idx]

    # Assert facets_cells integrity
    for cell_gid, edge_gids in enumerate(mesh.cells["facets"]):
        for edge_gid in edge_gids:
            idx = mesh.facets_cells_idx[edge_gid]
            if mesh.is_boundary_facet[edge_gid]:
                assert cell_gid == mesh.facets_cells["boundary"][1, idx]
            else:
                assert cell_gid in mesh.facets_cells["interior"][1:3, idx]

    # make sure the edges are opposite of the points
    for cell_gid, (point_ids, edge_ids) in enumerate(
        zip(mesh.cells["points"], mesh.cells["facets"])
    ):
        for k in range(len(point_ids)):
            assert set(point_ids) == set(
                [*mesh.edges["points"][edge_ids][k], point_ids[k]]
            )

    # make sure the is_boundary_point/edge/cell is consistent
    ref_cells = np.any(mesh.is_boundary_facet_local, axis=0)
    print(mesh.points)
    print(mesh.cells)
    assert np.all(mesh.is_boundary_cell == ref_cells)
    ref_points = np.zeros(len(mesh.points), dtype=bool)
    ref_points[mesh.idx[1][:, mesh.is_boundary_facet_local]] = True
    assert np.all(mesh.is_boundary_point == ref_points)

    assert len(mesh.control_volumes) == len(mesh.points)
    assert len(mesh.control_volume_centroids) == len(mesh.points)

    # TODO add more consistency checks

def compute_all_entities(mesh):
    mesh.is_boundary_point
    mesh.is_interior_point
    mesh.is_boundary_facet_local
    mesh.is_boundary_facet
    mesh.is_boundary_cell
    mesh.cell_volumes
    mesh.ce_ratios
    mesh.signed_cell_volumes
    mesh.cell_centroids
    mesh.control_volumes
    mesh.create_facets()
    mesh.facets_cells
    mesh.facets_cells_idx
    mesh.boundary_facets
    mesh.interior_facets
    mesh.cell_circumcenters
    mesh.signed_circumcenter_distances
    mesh.control_volume_centroids

    assert mesh._cv_cell_mask is not None
    assert mesh.edges is not None
    assert mesh.subdomains is not {}
    assert mesh._is_interior_point is not None
    assert mesh._is_boundary_point is not None
    assert mesh._is_boundary_facet_local is not None
    assert mesh._is_boundary_facet is not None
    assert mesh._is_boundary_cell is not None
    assert mesh._facets_cells is not None
    assert mesh._facets_cells_idx is not None
    assert mesh._boundary_facets is not None
    assert mesh._interior_facets is not None
    assert mesh._is_point_used is not None
    assert mesh._half_edge_coords is not None
    assert mesh._ei_dot_ei is not None
    assert mesh._volumes is not None
    assert mesh._ce_ratios is not None
    assert mesh._circumcenters is not None
    assert mesh._circumcenter_facet_distances is not None
    assert mesh._signed_circumcenter_distances is not None
    assert mesh._control_volumes is not None
    assert mesh._cell_partitions is not None
    assert mesh._cv_centroids is not None
    assert mesh._cvc_cell_mask is not None
    assert mesh._signed_cell_volumes is not None
    assert mesh._cell_centroids is not None

def assert_mesh_equality(mesh0, mesh1):
    assert np.all(mesh0.cells["points"] == mesh1.cells["points"])
    assert np.all(np.abs(mesh0.points - mesh1.points) < 1.0e-14)

    assert np.all(np.abs(mesh0.points - mesh1.points) < 1.0e-14)
    assert np.all(mesh0.edges["points"] == mesh1.edges["points"])

    # # Assume that in mesh1, the rows are ordered such that the edge indices [0] are in
    # # order. The mesh0 array isn't order in many cases, e.g., if cells were removed and
    # # rows appened the boundary array. Hence, justsort the array in mesh0 and compare.
    # k = np.argsort(mesh0.facets_cells["boundary"][0])
    # assert np.all(mesh0.facets_cells["boundary"][:, k] == mesh1.facets_cells["boundary"])

    # # The interior facets_cells are already ordered, even after remove_cells(). (As
    # # opposed to boundary edges, there can be no new interior edges, just some are
    # # removed.)
    # print(mesh0.facets_cells["interior"].T)
    # print()
    # print(mesh1.facets_cells["interior"].T)
    # assert np.all(mesh0.facets_cells["interior"] == mesh1.facets_cells["interior"])
    # exit(1)
    # # print()
    # # print(mesh0.facets_cells["interior"])
    # # print()
    # # print(mesh1.facets_cells["interior"])
    #
    # # These should not be equal; see reordering above
    # assert np.all(mesh0.facets_cells_idx == mesh1.facets_cells_idx)

    assert np.all(mesh0.boundary_facets == mesh1.boundary_facets)
    assert np.all(mesh0.interior_facets == mesh1.interior_facets)

    assert np.all(mesh0.is_point_used == mesh1.is_point_used)
    assert np.all(mesh0.is_boundary_point == mesh1.is_boundary_point)
    assert np.all(mesh0.is_interior_point == mesh1.is_interior_point)
    assert np.all(mesh0.is_boundary_facet_local == mesh1.is_boundary_facet_local)
    assert np.all(mesh0.is_boundary_facet == mesh1.is_boundary_facet)
    assert np.all(mesh0.is_boundary_cell == mesh1.is_boundary_cell)

    assert np.all(np.abs(mesh0.ei_dot_ei - mesh1.ei_dot_ei) < 1.0e-14)
    assert np.all(np.abs(mesh0.cell_volumes - mesh1.cell_volumes) < 1.0e-14)
    assert np.all(np.abs(mesh0.ce_ratios - mesh1.ce_ratios) < 1.0e-14)
    assert np.all(
        np.abs(mesh0.circumcenter_facet_distances - mesh1.circumcenter_facet_distances)
        < 1.0e-14
    )
    assert np.all(
        np.abs(
            mesh0.signed_circumcenter_distances - mesh1.signed_circumcenter_distances
        )
        < 1.0e-14
    )
    assert np.all(
        np.abs(mesh0.signed_cell_volumes - mesh1.signed_cell_volumes) < 1.0e-14
    )
    assert np.all(np.abs(mesh0.cell_centroids - mesh1.cell_centroids) < 1.0e-14)
    assert np.all(np.abs(mesh0.cell_circumcenters - mesh1.cell_circumcenters) < 1.0e-14)
    assert np.all(np.abs(mesh0.control_volumes - mesh1.control_volumes) < 1.0e-14)

    ipu = mesh0.is_point_used
    assert np.all(
        np.abs(
            mesh0.control_volume_centroids[ipu] - mesh1.control_volume_centroids[ipu]
        )
        < 1.0e-14
    )
<|endoftext|>


<commit_before># Stupid complex SQL because edge cases make it very hard to accurately
# retrieve the time someone logged in. This gets pretty close and I'll
# be cleaning it up over the next couple days.
online_now =  """
			--Find which players are online
			WITH online_players AS (
					SELECT se1.player_name, se1.time, se1.online FROM skynet_events se1
						INNER JOIN (
							SELECT MAX(time) AS time, player_name FROM skynet_events
							GROUP BY player_name
						) se2
					ON (se1.player_name = se2.player_name AND se1.time = se2.time AND se1.online = True)
			),
			--Get the log in times
				log_in AS (
					SELECT se1.time, se1.player_name FROM skynet_events se1
						INNER JOIN (
							SELECT player_name, time FROM (
								SELECT se2.player_name, se2.time FROM (
									SELECT MAX(se3.time) AS time, se3.player_name FROM (
										SELECT time, player_name, online FROM skynet_events WHERE online = False) se3
									GROUP BY player_name) se2
									INNER JOIN (
										SELECT player_name FROM online_players
									) AS se4
								ON (se2.player_name = se4.player_name)
							)
						) se5
					ON (se1.player_name = se5.player_name) WHERE se1.time>se5.time
			),
			--Yet Another With clause, grabs players who have never logged before
				yaw AS (
					SELECT se1.player_name, se1.time FROM log_in se1
					UNION SELECT se2.player_name, se2.time FROM skynet_events se2
						INNER JOIN (
							SELECT se3.player_name FROM online_players se3
								EXCEPT SELECT se4.player_name FROM online_log se4) se5
					ON (se2.player_name = se5.player_name)
			)
				SELECT MIN(time) AS time, player_name FROM yaw GROUP BY player_name;
			"""

online_at = """
			WITH at_time AS (
					SELECT time, player_name, online FROM skynet_events WHERE time<=%s
			),
				online_players AS (
					SELECT se1.player_name, se1.time, se1.online FROM at_time se1
						INNER JOIN (
							SELECT MAX(time) AS time, player_name FROM at_time
							GROUP BY player_name
						) se2
					ON (se1.player_name = se2.player_name AND se1.time = se2.time AND se1.online = True)
			),
				online_log AS (
					SELECT se1.player_name, se1.time FROM (
						SELECT MAX(se3.time) AS time, se3.player_name FROM (
							SELECT time, player_name, online FROM at_time WHERE online = False) se3
						GROUP BY player_name) se1
						INNER JOIN (
							SELECT player_name FROM online_players
						) se2
					ON (se1.player_name = se2.player_name)
			),
				log_in AS (
					SELECT se1.time, se1.player_name FROM at_time se1
						INNER JOIN (
							SELECT player_name, time FROM online_log
						) se2
					ON (se1.player_name = se2.player_name) WHERE se1.time>se2.time
			),
				yaw AS (
					SELECT se1.player_name, se1.time FROM log_in se1
					UNION SELECT se2.player_name, se2.time FROM at_time se2
						INNER JOIN (
							SELECT se3.player_name FROM online_players se3
								EXCEPT SELECT se4.player_name FROM online_log se4) se5
					ON (se2.player_name = se5.player_name)
			)
				SELECT MIN(time) AS time, player_name FROM yaw GROUP BY player_name;
			"""
<commit_msg>SQL Cleanup<commit_after># Stupid complex SQL because edge cases make it very hard to accurately
# retrieve the time someone logged in. This gets pretty close
online_now =  """
			--Find which players are online
			WITH online_players AS (
					SELECT se1.player_name, se1.time, se1.online FROM skynet_events se1
						INNER JOIN (
							SELECT MAX(time) AS time, player_name FROM skynet_events
							GROUP BY player_name
						) se2
					ON (se1.player_name = se2.player_name AND se1.time = se2.time AND se1.online = True)
			),
			--Get the log in times based on last logout
				log_in AS (
					SELECT se1.time, se1.player_name FROM skynet_events se1
						INNER JOIN (
							SELECT se2.player_name AS player_name, se2.time AS time FROM (
								SELECT MAX(se3.time) AS time, se3.player_name FROM (
									SELECT time, player_name, online FROM skynet_events WHERE online = False) se3
								GROUP BY player_name) se2
								INNER JOIN (
									SELECT player_name FROM online_players
								) se4
							ON (se2.player_name = se4.player_name)
						) se5
					ON (se1.player_name = se5.player_name) WHERE se1.time>se5.time
			)
			--Grabs all the logins including players who have never logged out before
				SELECT MIN(se1.time) AS time, se1.player_name FROM (
					SELECT se2.player_name as player_name, se2.time as time FROM log_in se2
					UNION SELECT se3.player_name, se3.time FROM skynet_events se3
						INNER JOIN (
							SELECT se4.player_name FROM online_players se4
								EXCEPT SELECT se5.player_name FROM log_in se5) se6
					ON (se3.player_name = se6.player_name)
					) se1 GROUP BY player_name;
			"""

online_at = """
			WITH at_time AS (
					SELECT time, player_name, online FROM skynet_events WHERE time<=%s
			),
				online_players AS (
					SELECT se1.player_name, se1.time, se1.online FROM at_time se1
						INNER JOIN (
							SELECT MAX(time) AS time, player_name FROM at_time
							GROUP BY player_name
						) se2
					ON (se1.player_name = se2.player_name AND se1.time = se2.time AND se1.online = True)
			),
				log_in AS (
					SELECT se1.time, se1.player_name FROM at_time se1
						INNER JOIN (
							SELECT se2.player_name AS player_name, se2.time AS time FROM (
								SELECT MAX(se3.time) AS time, se3.player_name FROM (
									SELECT time, player_name, online FROM at_time WHERE online = False) se3
								GROUP BY player_name) se2
								INNER JOIN (
									SELECT player_name FROM online_players
								) se4
							ON (se2.player_name = se4.player_name)
						) se5
					ON (se1.player_name = se5.player_name) WHERE se1.time>se5.time
			)
				SELECT MIN(se1.time) AS time, se1.player_name FROM (
					SELECT se2.player_name as player_name, se2.time as time FROM log_in se2
					UNION SELECT se3.player_name, se3.time FROM at_time se3
						INNER JOIN (
							SELECT se4.player_name FROM online_players se4
								EXCEPT SELECT se5.player_name FROM log_in se5) se6
					ON (se3.player_name = se6.player_name)
					) se1 GROUP BY player_name;
			"""
<|endoftext|>


<commit_before>import argparse
import subprocess
import sys

CLUSTER_SETTINGS = '" -q all.q -V -cwd -e ./log -o ./log {params.sge_opts} -w n -S /bin/bash"'
CLUSTER_FLAG = ("--drmaa", CLUSTER_SETTINGS, "-w", "30")

def _build_prefix(args):
    prefix = ["snakemake", "-pq"]
    if args.distribute:
        prefix.extend(CLUSTER_FLAG)

    return tuple(prefix)

def index(args):
    prefix = _build_prefix(args)
    args = prefix + ("prepare_reference", "--config", "reference=%s" % args.reference)
    args = " ".join(args)
    print args
    return subprocess.call(args, shell=True)

def align(args):
    print "Align reads"

def call(args):
    print "Call variants"

def genotype(args):
    print "Genotype"

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--distribute", action="store_true", help="Distribute analysis to Grid Engine-style cluster")
    subparsers = parser.add_subparsers()

    # Index a reference for use by BLASR.
    parser_index = subparsers.add_parser("index", help="index a reference sequence for use by BLASR")
    parser_index.add_argument("reference", help="FASTA file of reference to index")
    parser_index.set_defaults(func=index)

    # Align PacBio reads to an indexed reference with BLASR.
    parser_align = subparsers.add_parser("align", help="align PacBio reads to an indexed reference with BLASR")
    parser_align.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_align.add_argument("reads", help="text file with one absolute path to a PacBio reads file (.bax.h5) per line")
    parser_align.add_argument("--alignments", help="text file with one absolute path to a BLASR alignments file (.bam) per line", default="alignments.fofn")
    parser_align.add_argument("--alignments_dir", help="absolute path of directory for BLASR alignment files", default="alignments")
    parser_align.set_defaults(func=align)

    # Call SVs and indels from BLASR alignments.
    parser_caller = subparsers.add_parser("call", help="call SVs and indels by local assembly of BLASR-aligned reads")
    parser_caller.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_caller.add_argument("alignments", help="text file with one absolute path to a BLASR alignments file (.bam) per line")
    parser_caller.add_argument("variants", help="VCF of variants called by local assembly alignments")
    parser_caller.set_defaults(func=call)

    # Genotype SVs with Illumina reads.
    parser_genotyper = subparsers.add_parser("genotype", help="Genotype SVs with Illumina reads")
    parser_genotyper.add_argument("variants", help="VCF of SMRT SV variants to genotype")
    parser_genotyper.add_argument("genotyped_variants", help="VCF of SMRT SV variant genotypes for the given sample-level BAMs")
    parser_genotyper.add_argument("samples", nargs="+", help="one or more sample-level BAMs to genotype for the given variants")
    parser_genotyper.set_defaults(func=genotype)

    args = parser.parse_args()
    return_code = args.func(args)
    sys.exit(return_code)
<commit_msg>added code to run BLASR alignments.<commit_after>import argparse
import subprocess
import sys

CLUSTER_SETTINGS = '" -q all.q -V -cwd -e ./log -o ./log {params.sge_opts} -w n -S /bin/bash"'
CLUSTER_FLAG = ("--drmaa", CLUSTER_SETTINGS, "-w", "30")

def _build_prefix(args):
    prefix = ["snakemake", "-pq"]
    if args.distribute:
        prefix.extend(CLUSTER_FLAG)

    return tuple(prefix)

def index(args):
    prefix = _build_prefix(args)
    args = prefix + ("prepare_reference", "--config", "reference=%s" % args.reference)
    args = " ".join(args)
    return subprocess.call(args, shell=True)

def align(args):
    print "Align reads"
    prefix = _build_prefix(args)
    args = prefix + (
        "align_reads",
        "--config",
        "reference=%s" % args.reference,
        "reads=%s" % args.reads,
        "alignments=%s" % args.alignments,
        "alignments_dir=%s" % args.alignments_dir
    )
    args = " ".join(args)
    return subprocess.call(args, shell=True)

def call(args):
    print "Call variants"

def genotype(args):
    print "Genotype"

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--distribute", action="store_true", help="Distribute analysis to Grid Engine-style cluster")
    subparsers = parser.add_subparsers()

    # Index a reference for use by BLASR.
    parser_index = subparsers.add_parser("index", help="index a reference sequence for use by BLASR")
    parser_index.add_argument("reference", help="FASTA file of reference to index")
    parser_index.set_defaults(func=index)

    # Align PacBio reads to an indexed reference with BLASR.
    parser_align = subparsers.add_parser("align", help="align PacBio reads to an indexed reference with BLASR")
    parser_align.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_align.add_argument("reads", help="text file with one absolute path to a PacBio reads file (.bax.h5) per line")
    parser_align.add_argument("--alignments", help="text file with one absolute path to a BLASR alignments file (.bam) per line", default="alignments.fofn")
    parser_align.add_argument("--alignments_dir", help="absolute path of directory for BLASR alignment files", default="alignments")
    parser_align.set_defaults(func=align)

    # Call SVs and indels from BLASR alignments.
    parser_caller = subparsers.add_parser("call", help="call SVs and indels by local assembly of BLASR-aligned reads")
    parser_caller.add_argument("reference", help="FASTA file of indexed reference with .ctab and .sa in the same directory")
    parser_caller.add_argument("alignments", help="text file with one absolute path to a BLASR alignments file (.bam) per line")
    parser_caller.add_argument("variants", help="VCF of variants called by local assembly alignments")
    parser_caller.set_defaults(func=call)

    # Genotype SVs with Illumina reads.
    parser_genotyper = subparsers.add_parser("genotype", help="Genotype SVs with Illumina reads")
    parser_genotyper.add_argument("variants", help="VCF of SMRT SV variants to genotype")
    parser_genotyper.add_argument("genotyped_variants", help="VCF of SMRT SV variant genotypes for the given sample-level BAMs")
    parser_genotyper.add_argument("samples", nargs="+", help="one or more sample-level BAMs to genotype for the given variants")
    parser_genotyper.set_defaults(func=genotype)

    args = parser.parse_args()
    return_code = args.func(args)
    sys.exit(return_code)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import os

import click

DOT = "\u2603"
DASH = "\u26C4\u26C4\u26C4"

MORSE = {
    'a': (DOT, DASH),
    'b': (DASH, DOT, DOT, DOT),
    'c': (DASH, DOT, DASH, DOT),
    'd': (DASH, DOT, DOT),
    'e': (DOT,),
    'f': (DOT, DOT, DASH, DOT),
    'g': (DASH, DASH, DOT),
    'h': (DOT, DOT, DOT, DOT),
    'i': (DOT, DOT),
    'j': (DOT, DASH, DASH, DASH),
    'k': (DASH, DOT, DASH),
    'l': (DOT, DASH, DOT, DOT),
    'm': (DASH, DASH),
    'n': (DASH, DOT),
    'o': (DASH, DASH, DASH),
    'p': (DOT, DASH, DASH, DOT),
    'q': (DASH, DASH, DOT, DASH),
    'r': (DOT, DASH, DOT),
    's': (DOT, DOT, DOT),
    't': (DASH,),
    'u': (DOT, DOT, DASH),
    'v': (DOT, DOT, DOT, DASH),
    'w': (DOT, DASH, DASH),
    'x': (DASH, DOT, DOT, DASH),
    'y': (DASH, DOT, DASH, DASH),
    'z': (DASH, DASH, DOT, DOT),
    '1': (DOT, DASH, DASH, DASH, DASH),
    '2': (DOT, DOT, DASH, DASH, DASH),
    '3': (DOT, DOT, DOT, DASH, DASH),
    '4': (DOT, DOT, DOT, DOT, DASH),
    '5': (DOT, DOT, DOT, DOT, DOT),
    '6': (DASH, DOT, DOT, DOT, DOT),
    '7': (DASH, DASH, DOT, DOT, DOT),
    '8': (DASH, DASH, DASH, DOT, DOT),
    '9': (DASH, DASH, DASH, DASH, DOT),
    '0': (DASH, DASH, DASH, DASH, DASH),
}

UNMORSE = {v: k for k, v in MORSE.items()}

LETTER_SPACING = '   '
CHARACTER_SPACING = ' '
WORD_SPACING = '       '

def word_to_morse(word):
    return LETTER_SPACING.join(map(
        lambda c: CHARACTER_SPACING.join(MORSE.get(c.lower(), [c])),
        word,
    ))

def snorse(text):
    words = filter(bool, text.split(' '))
    return WORD_SPACING.join(map(
        word_to_morse,
        words,
    ))

def morse_to_word(morse):
    letters = morse.split(LETTER_SPACING)
    return ''.join(map(
        lambda c: UNMORSE.get(tuple(c.split(CHARACTER_SPACING)), c),
        letters,
    ))

def desnorse(text):
    words = filter(bool, text.split(WORD_SPACING))
    return ' '.join(map(
        morse_to_word,
        words,
    ))

@click.command()
@click.argument('text', required=False)
@click.option('-f', '--file', type=click.File())
def snorse_cli(text=None, file=None):
    if file is not None:
        text = file.read()
    elif text is None:
        text = click.get_text_stream('stdin').read()
    elif os.path.exists(os.path.expanduser(text)):
        text = open(os.path.expanduser(text)).read()
    click.echo(snorse(text))

@click.command()
@click.argument('text', required=False)
@click.option('-f', '--file', type=click.File())
def desnorse_cli(text=None, file=None):
    if file is not None:
        text = file.read()
    elif text is None:
        text = click.get_text_stream('stdin').read()
    elif os.path.exists(os.path.expanduser(text)):
        text = open(os.path.expanduser(text)).read()
    click.echo(desnorse(text))
<commit_msg>remove dict comprehension for py26<commit_after>from __future__ import unicode_literals

import os

import click

DOT = "\u2603"
DASH = "\u26C4\u26C4\u26C4"

MORSE = {
    'a': (DOT, DASH),
    'b': (DASH, DOT, DOT, DOT),
    'c': (DASH, DOT, DASH, DOT),
    'd': (DASH, DOT, DOT),
    'e': (DOT,),
    'f': (DOT, DOT, DASH, DOT),
    'g': (DASH, DASH, DOT),
    'h': (DOT, DOT, DOT, DOT),
    'i': (DOT, DOT),
    'j': (DOT, DASH, DASH, DASH),
    'k': (DASH, DOT, DASH),
    'l': (DOT, DASH, DOT, DOT),
    'm': (DASH, DASH),
    'n': (DASH, DOT),
    'o': (DASH, DASH, DASH),
    'p': (DOT, DASH, DASH, DOT),
    'q': (DASH, DASH, DOT, DASH),
    'r': (DOT, DASH, DOT),
    's': (DOT, DOT, DOT),
    't': (DASH,),
    'u': (DOT, DOT, DASH),
    'v': (DOT, DOT, DOT, DASH),
    'w': (DOT, DASH, DASH),
    'x': (DASH, DOT, DOT, DASH),
    'y': (DASH, DOT, DASH, DASH),
    'z': (DASH, DASH, DOT, DOT),
    '1': (DOT, DASH, DASH, DASH, DASH),
    '2': (DOT, DOT, DASH, DASH, DASH),
    '3': (DOT, DOT, DOT, DASH, DASH),
    '4': (DOT, DOT, DOT, DOT, DASH),
    '5': (DOT, DOT, DOT, DOT, DOT),
    '6': (DASH, DOT, DOT, DOT, DOT),
    '7': (DASH, DASH, DOT, DOT, DOT),
    '8': (DASH, DASH, DASH, DOT, DOT),
    '9': (DASH, DASH, DASH, DASH, DOT),
    '0': (DASH, DASH, DASH, DASH, DASH),
}

UNMORSE = dict(
    (v, k) for k, v in MORSE.items()
)

LETTER_SPACING = '   '
CHARACTER_SPACING = ' '
WORD_SPACING = '       '

def word_to_morse(word):
    return LETTER_SPACING.join(map(
        lambda c: CHARACTER_SPACING.join(MORSE.get(c.lower(), [c])),
        word,
    ))

def snorse(text):
    words = filter(bool, text.split(' '))
    return WORD_SPACING.join(map(
        word_to_morse,
        words,
    ))

def morse_to_word(morse):
    letters = morse.split(LETTER_SPACING)
    return ''.join(map(
        lambda c: UNMORSE.get(tuple(c.split(CHARACTER_SPACING)), c),
        letters,
    ))

def desnorse(text):
    words = filter(bool, text.split(WORD_SPACING))
    return ' '.join(map(
        morse_to_word,
        words,
    ))

@click.command()
@click.argument('text', required=False)
@click.option('-f', '--file', type=click.File())
def snorse_cli(text=None, file=None):
    if file is not None:
        text = file.read()
    elif text is None:
        text = click.get_text_stream('stdin').read()
    elif os.path.exists(os.path.expanduser(text)):
        text = open(os.path.expanduser(text)).read()
    click.echo(snorse(text))

@click.command()
@click.argument('text', required=False)
@click.option('-f', '--file', type=click.File())
def desnorse_cli(text=None, file=None):
    if file is not None:
        text = file.read()
    elif text is None:
        text = click.get_text_stream('stdin').read()
    elif os.path.exists(os.path.expanduser(text)):
        text = open(os.path.expanduser(text)).read()
    click.echo(desnorse(text))
<|endoftext|>


<commit_before>#!/usr/bin/python
#

# Copyright (C) 2009 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Script for unittesting documentation"""

import unittest
import re
import itertools
import operator

from ganeti import _constants
from ganeti import utils
from ganeti import cmdlib
from ganeti import build
from ganeti import compat
from ganeti import mcpu
from ganeti import opcodes
from ganeti import constants
from ganeti.rapi import baserlib
from ganeti.rapi import rlib2
from ganeti.rapi import connector

import testutils

VALID_URI_RE = re.compile(r"^[-/a-z0-9]*$")

RAPI_OPCODE_EXCLUDE = compat.UniqueFrozenset([
  # Not yet implemented
  opcodes.OpBackupRemove,
  opcodes.OpClusterConfigQuery,
  opcodes.OpClusterRepairDiskSizes,
  opcodes.OpClusterVerify,
  opcodes.OpClusterVerifyDisks,
  opcodes.OpInstanceChangeGroup,
  opcodes.OpInstanceMove,
  opcodes.OpNodeQueryvols,
  opcodes.OpOobCommand,
  opcodes.OpTagsSearch,
  opcodes.OpClusterActivateMasterIp,
  opcodes.OpClusterDeactivateMasterIp,
  opcodes.OpExtStorageDiagnose,

  # Difficult if not impossible
  opcodes.OpClusterDestroy,
  opcodes.OpClusterPostInit,
  opcodes.OpClusterRename,
  opcodes.OpNodeAdd,
  opcodes.OpNodeRemove,

  # Very sensitive in nature
  opcodes.OpRestrictedCommand,
  opcodes.OpClusterRenewCrypto,

  # Helper opcodes (e.g. submitted by LUs)
  opcodes.OpClusterVerifyConfig,
  opcodes.OpClusterVerifyGroup,
  opcodes.OpGroupEvacuate,
  opcodes.OpGroupVerifyDisks,

  # Test opcodes
  opcodes.OpTestAllocator,
  opcodes.OpTestDelay,
  opcodes.OpTestDummy,
  opcodes.OpTestJqueue,
  ])

def _ReadDocFile(filename):
  return utils.ReadFile("%s/doc/%s" %
                        (testutils.GetSourceDir(), filename))

class TestHooksDocs(unittest.TestCase):
  HOOK_PATH_OK = compat.UniqueFrozenset([
    "master-ip-turnup",
    "master-ip-turndown",
    ])

  def test(self):
    """Check whether all hooks are documented.

    """
    hooksdoc = _ReadDocFile("hooks.rst")

    # Reverse mapping from LU to opcode
    lu2opcode = dict((lu, op)
                     for (op, lu) in mcpu.Processor.DISPATCH_TABLE.items())
    assert len(lu2opcode) == len(mcpu.Processor.DISPATCH_TABLE), \
      "Found duplicate entries"

    hooks_paths = frozenset(re.findall("^:directory:\s*(.+)\s*$", hooksdoc,
                                       re.M))
    self.assertTrue(self.HOOK_PATH_OK.issubset(hooks_paths),
                    msg="Whitelisted path not found in documentation")

    raw_hooks_ops = re.findall("^OP_(?!CODE$).+$", hooksdoc, re.M)
    hooks_ops = set()
    duplicate_ops = set()
    for op in raw_hooks_ops:
      if op in hooks_ops:
        duplicate_ops.add(op)
      else:
        hooks_ops.add(op)

    self.assertFalse(duplicate_ops,
                     msg="Found duplicate opcode documentation: %s" %
                         utils.CommaJoin(duplicate_ops))

    seen_paths = set()
    seen_ops = set()

    self.assertFalse(duplicate_ops,
                     msg="Found duplicated hook documentation: %s" %
                         utils.CommaJoin(duplicate_ops))

    for name in dir(cmdlib):
      lucls = getattr(cmdlib, name)

      if (isinstance(lucls, type) and
          issubclass(lucls, cmdlib.LogicalUnit) and
          hasattr(lucls, "HPATH")):
        if lucls.HTYPE is None:
          continue

        opcls = lu2opcode.get(lucls, None)

        if opcls:
          seen_ops.add(opcls.OP_ID)
          self.assertTrue(opcls.OP_ID in hooks_ops,
                          msg="Missing hook documentation for %s" %
                              opcls.OP_ID)
        self.assertTrue(lucls.HPATH in hooks_paths,
                        msg="Missing documentation for hook %s/%s" %
                            (lucls.HTYPE, lucls.HPATH))
        seen_paths.add(lucls.HPATH)

    missed_ops = hooks_ops - seen_ops
    missed_paths = hooks_paths - seen_paths - self.HOOK_PATH_OK

    self.assertFalse(missed_ops,
                     msg="Op documents hook not existing anymore: %s" %
                         utils.CommaJoin(missed_ops))

    self.assertFalse(missed_paths,
                     msg="Hook path does not exist in opcode: %s" %
                         utils.CommaJoin(missed_paths))

class TestRapiDocs(unittest.TestCase):
  def _CheckRapiResource(self, uri, fixup, handler):
    docline = "%s resource." % uri
    self.assertEqual(handler.__doc__.splitlines()[0].strip(), docline,
                     msg=("First line of %r's docstring is not %r" %
                          (handler, docline)))

    # Apply fixes before testing
    for (rx, value) in fixup.items():
      uri = rx.sub(value, uri)

    self.assertTrue(VALID_URI_RE.match(uri), msg="Invalid URI %r" % uri)

  def test(self):
    """Check whether all RAPI resources are documented.

    """
    rapidoc = _ReadDocFile("rapi.rst")

    node_name = re.escape("[node_name]")
    instance_name = re.escape("[instance_name]")
    group_name = re.escape("[group_name]")
    network_name = re.escape("[network_name]")
    job_id = re.escape("[job_id]")
    disk_index = re.escape("[disk_index]")
    filter_uuid = re.escape("[filter_uuid]")
    query_res = re.escape("[resource]")

    resources = connector.GetHandlers(node_name, instance_name,
                                      group_name, network_name,
                                      job_id, disk_index, filter_uuid,
                                      query_res)

    handler_dups = utils.FindDuplicates(resources.values())
    self.assertFalse(handler_dups,
                     msg=("Resource handlers used more than once: %r" %
                          handler_dups))

    uri_check_fixup = {
      re.compile(node_name): "node1examplecom",
      re.compile(instance_name): "inst1examplecom",
      re.compile(group_name): "group4440",
      re.compile(network_name): "network5550",
      re.compile(job_id): "9409",
      re.compile(disk_index): "123",
      re.compile(filter_uuid): "c863fbb5-f248-47bf-869b-cea259890061",
      re.compile(query_res): "lock",
      }

    assert compat.all(VALID_URI_RE.match(value)
                      for value in uri_check_fixup.values()), \
           "Fixup values must be valid URIs, too"

    titles = []

    prevline = None
    for line in rapidoc.splitlines():
      if re.match(r"^\++$", line):
        titles.append(prevline)

      prevline = line

    prefix_exception = compat.UniqueFrozenset(["/", "/version", "/2"])

    undocumented = []
    used_uris = []

    for key, handler in resources.iteritems():
      # Regex objects
      if hasattr(key, "match"):
        self.assert_(key.pattern.startswith("^/2/"),
                     msg="Pattern %r does not start with '^/2/'" % key.pattern)
        self.assertEqual(key.pattern[-1], "$")

        found = False
        for title in titles:
          if title.startswith("``") and title.endswith("``"):
            uri = title[2:-2]
            if key.match(uri):
              self._CheckRapiResource(uri, uri_check_fixup, handler)
              used_uris.append(uri)
              found = True
              break

        if not found:
          # TODO: Find better way of identifying resource
          undocumented.append(key.pattern)

      else:
        self.assert_(key.startswith("/2/") or key in prefix_exception,
                     msg="Path %r does not start with '/2/'" % key)

        if ("``%s``" % key) in titles:
          self._CheckRapiResource(key, {}, handler)
          used_uris.append(key)
        else:
          undocumented.append(key)

    self.failIf(undocumented,
                msg=("Missing RAPI resource documentation for %s" %
                     utils.CommaJoin(undocumented)))

    uri_dups = utils.FindDuplicates(used_uris)
    self.failIf(uri_dups,
                msg=("URIs matched by more than one resource: %s" %
                     utils.CommaJoin(uri_dups)))

    self._FindRapiMissing(resources.values())
    self._CheckTagHandlers(resources.values())

  def _FindRapiMissing(self, handlers):
    used = frozenset(itertools.chain(*map(baserlib.GetResourceOpcodes,
                                          handlers)))

    unexpected = used & RAPI_OPCODE_EXCLUDE
    self.assertFalse(unexpected,
      msg=("Found RAPI resources for excluded opcodes: %s" %
           utils.CommaJoin(_GetOpIds(unexpected))))

    missing = (frozenset(opcodes.OP_MAPPING.values()) - used -
               RAPI_OPCODE_EXCLUDE)
    self.assertFalse(missing,
      msg=("Missing RAPI resources for opcodes: %s" %
           utils.CommaJoin(_GetOpIds(missing))))

  def _CheckTagHandlers(self, handlers):
    tag_handlers = filter(lambda x: issubclass(x, rlib2._R_Tags), handlers)
    self.assertEqual(frozenset(map(operator.attrgetter("TAG_LEVEL"),
                                   tag_handlers)),
                     constants.VALID_TAG_TYPES)

def _GetOpIds(ops):
  """Returns C{OP_ID} for all opcodes in passed sequence.

  """
  return sorted(opcls.OP_ID for opcls in ops)

class TestManpages(unittest.TestCase):
  """Manpage tests"""

  @staticmethod
  def _ReadManFile(name):
    return utils.ReadFile("%s/man/%s.rst" %
                          (testutils.GetSourceDir(), name))

  @staticmethod
  def _LoadScript(name):
    return build.LoadModule("scripts/%s" % name)

  def test(self):
    for script in _constants.GNT_SCRIPTS:
      self._CheckManpage(script,
                         self._ReadManFile(script),
                         self._LoadScript(script).commands.keys())

  def _CheckManpage(self, script, mantext, commands):
    missing = []

    for cmd in commands:
      pattern = r"^(\| )?\*\*%s\*\*" % re.escape(cmd)
      if not re.findall(pattern, mantext, re.DOTALL | re.MULTILINE):
        missing.append(cmd)

    self.failIf(missing,
                msg=("Manpage for '%s' missing documentation for %s" %
                     (script, utils.CommaJoin(missing))))

if __name__ == "__main__":
  testutils.GanetiTestProgram()
<commit_msg>Exclude OpTestOsParams opcode from RAPI<commit_after>#!/usr/bin/python
#

# Copyright (C) 2009 Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Script for unittesting documentation"""

import unittest
import re
import itertools
import operator

from ganeti import _constants
from ganeti import utils
from ganeti import cmdlib
from ganeti import build
from ganeti import compat
from ganeti import mcpu
from ganeti import opcodes
from ganeti import constants
from ganeti.rapi import baserlib
from ganeti.rapi import rlib2
from ganeti.rapi import connector

import testutils

VALID_URI_RE = re.compile(r"^[-/a-z0-9]*$")

RAPI_OPCODE_EXCLUDE = compat.UniqueFrozenset([
  # Not yet implemented
  opcodes.OpBackupRemove,
  opcodes.OpClusterConfigQuery,
  opcodes.OpClusterRepairDiskSizes,
  opcodes.OpClusterVerify,
  opcodes.OpClusterVerifyDisks,
  opcodes.OpInstanceChangeGroup,
  opcodes.OpInstanceMove,
  opcodes.OpNodeQueryvols,
  opcodes.OpOobCommand,
  opcodes.OpTagsSearch,
  opcodes.OpClusterActivateMasterIp,
  opcodes.OpClusterDeactivateMasterIp,
  opcodes.OpExtStorageDiagnose,

  # Difficult if not impossible
  opcodes.OpClusterDestroy,
  opcodes.OpClusterPostInit,
  opcodes.OpClusterRename,
  opcodes.OpNodeAdd,
  opcodes.OpNodeRemove,

  # Very sensitive in nature
  opcodes.OpRestrictedCommand,
  opcodes.OpClusterRenewCrypto,

  # Helper opcodes (e.g. submitted by LUs)
  opcodes.OpClusterVerifyConfig,
  opcodes.OpClusterVerifyGroup,
  opcodes.OpGroupEvacuate,
  opcodes.OpGroupVerifyDisks,

  # Test opcodes
  opcodes.OpTestAllocator,
  opcodes.OpTestDelay,
  opcodes.OpTestDummy,
  opcodes.OpTestJqueue,
  opcodes.OpTestOsParams,
  ])

def _ReadDocFile(filename):
  return utils.ReadFile("%s/doc/%s" %
                        (testutils.GetSourceDir(), filename))

class TestHooksDocs(unittest.TestCase):
  HOOK_PATH_OK = compat.UniqueFrozenset([
    "master-ip-turnup",
    "master-ip-turndown",
    ])

  def test(self):
    """Check whether all hooks are documented.

    """
    hooksdoc = _ReadDocFile("hooks.rst")

    # Reverse mapping from LU to opcode
    lu2opcode = dict((lu, op)
                     for (op, lu) in mcpu.Processor.DISPATCH_TABLE.items())
    assert len(lu2opcode) == len(mcpu.Processor.DISPATCH_TABLE), \
      "Found duplicate entries"

    hooks_paths = frozenset(re.findall("^:directory:\s*(.+)\s*$", hooksdoc,
                                       re.M))
    self.assertTrue(self.HOOK_PATH_OK.issubset(hooks_paths),
                    msg="Whitelisted path not found in documentation")

    raw_hooks_ops = re.findall("^OP_(?!CODE$).+$", hooksdoc, re.M)
    hooks_ops = set()
    duplicate_ops = set()
    for op in raw_hooks_ops:
      if op in hooks_ops:
        duplicate_ops.add(op)
      else:
        hooks_ops.add(op)

    self.assertFalse(duplicate_ops,
                     msg="Found duplicate opcode documentation: %s" %
                         utils.CommaJoin(duplicate_ops))

    seen_paths = set()
    seen_ops = set()

    self.assertFalse(duplicate_ops,
                     msg="Found duplicated hook documentation: %s" %
                         utils.CommaJoin(duplicate_ops))

    for name in dir(cmdlib):
      lucls = getattr(cmdlib, name)

      if (isinstance(lucls, type) and
          issubclass(lucls, cmdlib.LogicalUnit) and
          hasattr(lucls, "HPATH")):
        if lucls.HTYPE is None:
          continue

        opcls = lu2opcode.get(lucls, None)

        if opcls:
          seen_ops.add(opcls.OP_ID)
          self.assertTrue(opcls.OP_ID in hooks_ops,
                          msg="Missing hook documentation for %s" %
                              opcls.OP_ID)
        self.assertTrue(lucls.HPATH in hooks_paths,
                        msg="Missing documentation for hook %s/%s" %
                            (lucls.HTYPE, lucls.HPATH))
        seen_paths.add(lucls.HPATH)

    missed_ops = hooks_ops - seen_ops
    missed_paths = hooks_paths - seen_paths - self.HOOK_PATH_OK

    self.assertFalse(missed_ops,
                     msg="Op documents hook not existing anymore: %s" %
                         utils.CommaJoin(missed_ops))

    self.assertFalse(missed_paths,
                     msg="Hook path does not exist in opcode: %s" %
                         utils.CommaJoin(missed_paths))

class TestRapiDocs(unittest.TestCase):
  def _CheckRapiResource(self, uri, fixup, handler):
    docline = "%s resource." % uri
    self.assertEqual(handler.__doc__.splitlines()[0].strip(), docline,
                     msg=("First line of %r's docstring is not %r" %
                          (handler, docline)))

    # Apply fixes before testing
    for (rx, value) in fixup.items():
      uri = rx.sub(value, uri)

    self.assertTrue(VALID_URI_RE.match(uri), msg="Invalid URI %r" % uri)

  def test(self):
    """Check whether all RAPI resources are documented.

    """
    rapidoc = _ReadDocFile("rapi.rst")

    node_name = re.escape("[node_name]")
    instance_name = re.escape("[instance_name]")
    group_name = re.escape("[group_name]")
    network_name = re.escape("[network_name]")
    job_id = re.escape("[job_id]")
    disk_index = re.escape("[disk_index]")
    filter_uuid = re.escape("[filter_uuid]")
    query_res = re.escape("[resource]")

    resources = connector.GetHandlers(node_name, instance_name,
                                      group_name, network_name,
                                      job_id, disk_index, filter_uuid,
                                      query_res)

    handler_dups = utils.FindDuplicates(resources.values())
    self.assertFalse(handler_dups,
                     msg=("Resource handlers used more than once: %r" %
                          handler_dups))

    uri_check_fixup = {
      re.compile(node_name): "node1examplecom",
      re.compile(instance_name): "inst1examplecom",
      re.compile(group_name): "group4440",
      re.compile(network_name): "network5550",
      re.compile(job_id): "9409",
      re.compile(disk_index): "123",
      re.compile(filter_uuid): "c863fbb5-f248-47bf-869b-cea259890061",
      re.compile(query_res): "lock",
      }

    assert compat.all(VALID_URI_RE.match(value)
                      for value in uri_check_fixup.values()), \
           "Fixup values must be valid URIs, too"

    titles = []

    prevline = None
    for line in rapidoc.splitlines():
      if re.match(r"^\++$", line):
        titles.append(prevline)

      prevline = line

    prefix_exception = compat.UniqueFrozenset(["/", "/version", "/2"])

    undocumented = []
    used_uris = []

    for key, handler in resources.iteritems():
      # Regex objects
      if hasattr(key, "match"):
        self.assert_(key.pattern.startswith("^/2/"),
                     msg="Pattern %r does not start with '^/2/'" % key.pattern)
        self.assertEqual(key.pattern[-1], "$")

        found = False
        for title in titles:
          if title.startswith("``") and title.endswith("``"):
            uri = title[2:-2]
            if key.match(uri):
              self._CheckRapiResource(uri, uri_check_fixup, handler)
              used_uris.append(uri)
              found = True
              break

        if not found:
          # TODO: Find better way of identifying resource
          undocumented.append(key.pattern)

      else:
        self.assert_(key.startswith("/2/") or key in prefix_exception,
                     msg="Path %r does not start with '/2/'" % key)

        if ("``%s``" % key) in titles:
          self._CheckRapiResource(key, {}, handler)
          used_uris.append(key)
        else:
          undocumented.append(key)

    self.failIf(undocumented,
                msg=("Missing RAPI resource documentation for %s" %
                     utils.CommaJoin(undocumented)))

    uri_dups = utils.FindDuplicates(used_uris)
    self.failIf(uri_dups,
                msg=("URIs matched by more than one resource: %s" %
                     utils.CommaJoin(uri_dups)))

    self._FindRapiMissing(resources.values())
    self._CheckTagHandlers(resources.values())

  def _FindRapiMissing(self, handlers):
    used = frozenset(itertools.chain(*map(baserlib.GetResourceOpcodes,
                                          handlers)))

    unexpected = used & RAPI_OPCODE_EXCLUDE
    self.assertFalse(unexpected,
      msg=("Found RAPI resources for excluded opcodes: %s" %
           utils.CommaJoin(_GetOpIds(unexpected))))

    missing = (frozenset(opcodes.OP_MAPPING.values()) - used -
               RAPI_OPCODE_EXCLUDE)
    self.assertFalse(missing,
      msg=("Missing RAPI resources for opcodes: %s" %
           utils.CommaJoin(_GetOpIds(missing))))

  def _CheckTagHandlers(self, handlers):
    tag_handlers = filter(lambda x: issubclass(x, rlib2._R_Tags), handlers)
    self.assertEqual(frozenset(map(operator.attrgetter("TAG_LEVEL"),
                                   tag_handlers)),
                     constants.VALID_TAG_TYPES)

def _GetOpIds(ops):
  """Returns C{OP_ID} for all opcodes in passed sequence.

  """
  return sorted(opcls.OP_ID for opcls in ops)

class TestManpages(unittest.TestCase):
  """Manpage tests"""

  @staticmethod
  def _ReadManFile(name):
    return utils.ReadFile("%s/man/%s.rst" %
                          (testutils.GetSourceDir(), name))

  @staticmethod
  def _LoadScript(name):
    return build.LoadModule("scripts/%s" % name)

  def test(self):
    for script in _constants.GNT_SCRIPTS:
      self._CheckManpage(script,
                         self._ReadManFile(script),
                         self._LoadScript(script).commands.keys())

  def _CheckManpage(self, script, mantext, commands):
    missing = []

    for cmd in commands:
      pattern = r"^(\| )?\*\*%s\*\*" % re.escape(cmd)
      if not re.findall(pattern, mantext, re.DOTALL | re.MULTILINE):
        missing.append(cmd)

    self.failIf(missing,
                msg=("Manpage for '%s' missing documentation for %s" %
                     (script, utils.CommaJoin(missing))))

if __name__ == "__main__":
  testutils.GanetiTestProgram()
<|endoftext|>


<commit_before>import unittest

from urlobject.query_string import QueryString

class QueryStringTest(unittest.TestCase):

    def test_preserves_equality_with_original_string(self):
        assert QueryString(u'abc=123') == u'abc=123'

    def test_preserves_hash_value_of_original_string(self):
        assert hash(QueryString(u'abc=123')) == hash(u'abc=123')

    def test_list_returns_an_empty_list_for_empty_QueryStrings(self):
        assert QueryString(u'').list == []

    def test_list_correctly_splits_on_ampersands(self):
        assert QueryString(u'abc=123&def=456').list == [
            (u'abc', u'123'), (u'def', u'456')]

    def test_list_correctly_splits_on_semicolons(self):
        assert QueryString(u'abc=123;def=456').list == [
            (u'abc', u'123'), (u'def', u'456')]

    def test_list_correctly_decodes_special_chars(self):
        assert QueryString(u'a%20b=c%20d').list == [(u'a b', u'c d')]
        assert QueryString(u'a+b=c+d').list == [(u'a b', u'c d')]
        assert (QueryString(u'my%20weird%20field=q1!2%22\'w%245%267%2Fz8)%3F').list ==
                [(u'my weird field', u'q1!2"\'w$5&7/z8)?')])

    def test_list_correctly_decodes_utf_8(self):
        assert QueryString(u'foo=%EF%BF%BD').list == [(u'foo', u'\ufffd')]

    def test_list_doesnt_split_on_percent_encoded_special_chars(self):
        assert QueryString(u'a%26b%3Dc%3F=a%26b%3Dc%3F').list == [
            (u'a&b=c?', u'a&b=c?')]

    def test_list_doesnt_break_if_two_parameters_have_the_same_name(self):
        assert QueryString(u'abc=123;abc=456').list == [
            (u'abc', u'123'), (u'abc', u'456')]

    def test_list_uses_none_as_the_value_for_valueless_parameters(self):
        assert QueryString(u'abc').list == [(u'abc', None)]
        assert QueryString(u'abc=123&def&ghi=456').list == [
            (u'abc', u'123'), (u'def', None), (u'ghi', u'456')]

    def test_list_uses_empty_string_for_empty_valued_parameters(self):
        assert QueryString(u'abc=').list == [(u'abc', u'')]
        assert QueryString(u'abc=123&def=&ghi=456').list == [
            (u'abc', u'123'), (u'def', u''), (u'ghi', u'456')]

    def test_list_uses_empty_string_for_anonymous_parameters(self):
        assert QueryString(u'=123').list == [(u'', u'123')]
        assert QueryString(u'abc=123&=456&ghi=789').list == [
            (u'abc', u'123'), (u'', u'456'), (u'ghi', u'789')]

    def test_list_can_handle_void_parameters(self):
        assert QueryString(u'abc=123&&def=456').list == [
            (u'abc', u'123'), (u'', None), (u'def', u'456')]
        assert QueryString(u'abc=123&=&def=456').list == [
            (u'abc', u'123'), (u'', u''), (u'def', u'456')]

    def test_dict_returns_a_dictionary_with_one_value_per_key(self):
        assert QueryString(u'abc=123&abc=456').dict == {u'abc': u'456'}

    def test_multi_dict_returns_a_dictionary_with_all_values_per_key(self):
        assert QueryString(u'abc=123&abc=456').multi_dict == {
            u'abc': [u'123', u'456']}

    def test_add_param_encodes_and_adds_the_given_parameter_to_the_QueryString(self):
        s = QueryString(u'')
        assert s.add_param(u'abc', u'123') == u'abc=123'
        assert (s.add_param(u'abc', u'123')
                 .add_param(u'def', u'456') == u'abc=123&def=456')

    def test_add_param_can_add_valueless_parameters(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'def', None) == u'abc=123&def'

    def test_add_param_can_add_empty_valued_parameters(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'def', u'') == u'abc=123&def='

    def test_add_param_can_add_anonymous_parameters(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'', u'456') == u'abc=123&=456'

    def test_add_param_encodes_utf8(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'foo', u'\ufffd') == u'abc=123&foo=%EF%BF%BD'

    def test_add_param_allows_the_same_parameter_name_to_be_added_twice(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'abc', u'456') == u'abc=123&abc=456'

    def test_add_param_encodes_special_characters(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'd e f', u'4+5#6') == u'abc=123&d%20e%20f=4%2B5%236'

    def test_set_param_replaces_existing_parameter_names(self):
        s = QueryString(u'abc=123&abc=456')
        assert s.set_param(u'abc', '789') == u'abc=789'

    def test_del_param_removes_all_instances_of_the_parameter_from_the_QueryString(self):
        s = QueryString(u'abc=123&def=456&abc=789')
        assert s.del_param(u'abc') == u'def=456'
        assert s.del_param(u'def') == u'abc=123&abc=789'

    def test_del_param_can_remove_valueless_parameters(self):
        valueless = QueryString(u'abc=123&def&abc=456')
        empty_valued = QueryString(u'abc=123&def=&abc=456')
        assert valueless.del_param(u'def') == u'abc=123&abc=456'
        assert empty_valued.del_param(u'def') == u'abc=123&abc=456'

    def test_del_param_can_remove_anonymous_parameters(self):
        s = QueryString(u'abc=123&=456&def=789')
        assert s.del_param(u'') == u'abc=123&def=789'
<commit_msg>Added tests for add/set/del_params.<commit_after>import unittest

from urlobject.query_string import QueryString

class QueryStringTest(unittest.TestCase):

    def test_preserves_equality_with_original_string(self):
        assert QueryString(u'abc=123') == u'abc=123'

    def test_preserves_hash_value_of_original_string(self):
        assert hash(QueryString(u'abc=123')) == hash(u'abc=123')

    def test_list_returns_an_empty_list_for_empty_QueryStrings(self):
        assert QueryString(u'').list == []

    def test_list_correctly_splits_on_ampersands(self):
        assert QueryString(u'abc=123&def=456').list == [
            (u'abc', u'123'), (u'def', u'456')]

    def test_list_correctly_splits_on_semicolons(self):
        assert QueryString(u'abc=123;def=456').list == [
            (u'abc', u'123'), (u'def', u'456')]

    def test_list_correctly_decodes_special_chars(self):
        assert QueryString(u'a%20b=c%20d').list == [(u'a b', u'c d')]
        assert QueryString(u'a+b=c+d').list == [(u'a b', u'c d')]
        assert (QueryString(u'my%20weird%20field=q1!2%22\'w%245%267%2Fz8)%3F').list ==
                [(u'my weird field', u'q1!2"\'w$5&7/z8)?')])

    def test_list_correctly_decodes_utf_8(self):
        assert QueryString(u'foo=%EF%BF%BD').list == [(u'foo', u'\ufffd')]

    def test_list_doesnt_split_on_percent_encoded_special_chars(self):
        assert QueryString(u'a%26b%3Dc%3F=a%26b%3Dc%3F').list == [
            (u'a&b=c?', u'a&b=c?')]

    def test_list_doesnt_break_if_two_parameters_have_the_same_name(self):
        assert QueryString(u'abc=123;abc=456').list == [
            (u'abc', u'123'), (u'abc', u'456')]

    def test_list_uses_none_as_the_value_for_valueless_parameters(self):
        assert QueryString(u'abc').list == [(u'abc', None)]
        assert QueryString(u'abc=123&def&ghi=456').list == [
            (u'abc', u'123'), (u'def', None), (u'ghi', u'456')]

    def test_list_uses_empty_string_for_empty_valued_parameters(self):
        assert QueryString(u'abc=').list == [(u'abc', u'')]
        assert QueryString(u'abc=123&def=&ghi=456').list == [
            (u'abc', u'123'), (u'def', u''), (u'ghi', u'456')]

    def test_list_uses_empty_string_for_anonymous_parameters(self):
        assert QueryString(u'=123').list == [(u'', u'123')]
        assert QueryString(u'abc=123&=456&ghi=789').list == [
            (u'abc', u'123'), (u'', u'456'), (u'ghi', u'789')]

    def test_list_can_handle_void_parameters(self):
        assert QueryString(u'abc=123&&def=456').list == [
            (u'abc', u'123'), (u'', None), (u'def', u'456')]
        assert QueryString(u'abc=123&=&def=456').list == [
            (u'abc', u'123'), (u'', u''), (u'def', u'456')]

    def test_dict_returns_a_dictionary_with_one_value_per_key(self):
        assert QueryString(u'abc=123&abc=456').dict == {u'abc': u'456'}

    def test_multi_dict_returns_a_dictionary_with_all_values_per_key(self):
        assert QueryString(u'abc=123&abc=456').multi_dict == {
            u'abc': [u'123', u'456']}

    def test_add_param_encodes_and_adds_the_given_parameter_to_the_QueryString(self):
        s = QueryString(u'')
        assert s.add_param(u'abc', u'123') == u'abc=123'
        assert (s.add_param(u'abc', u'123')
                 .add_param(u'def', u'456') == u'abc=123&def=456')

    def test_add_param_can_add_valueless_parameters(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'def', None) == u'abc=123&def'

    def test_add_param_can_add_empty_valued_parameters(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'def', u'') == u'abc=123&def='

    def test_add_param_can_add_anonymous_parameters(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'', u'456') == u'abc=123&=456'

    def test_add_param_encodes_utf8(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'foo', u'\ufffd') == u'abc=123&foo=%EF%BF%BD'

    def test_add_param_allows_the_same_parameter_name_to_be_added_twice(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'abc', u'456') == u'abc=123&abc=456'

    def test_add_param_encodes_special_characters(self):
        s = QueryString(u'abc=123')
        assert s.add_param(u'd e f', u'4+5#6') == u'abc=123&d%20e%20f=4%2B5%236'

    def test_set_param_replaces_existing_parameter_names(self):
        s = QueryString(u'abc=123&abc=456')
        assert s.set_param(u'abc', '789') == u'abc=789'

    def test_del_param_removes_all_instances_of_the_parameter_from_the_QueryString(self):
        s = QueryString(u'abc=123&def=456&abc=789')
        assert s.del_param(u'abc') == u'def=456'
        assert s.del_param(u'def') == u'abc=123&abc=789'

    def test_del_param_can_remove_valueless_parameters(self):
        valueless = QueryString(u'abc=123&def&abc=456')
        empty_valued = QueryString(u'abc=123&def=&abc=456')
        assert valueless.del_param(u'def') == u'abc=123&abc=456'
        assert empty_valued.del_param(u'def') == u'abc=123&abc=456'

    def test_del_param_can_remove_anonymous_parameters(self):
        s = QueryString(u'abc=123&=456&def=789')
        assert s.del_param(u'') == u'abc=123&def=789'

    def test_add_params_is_equivalent_to_calling_add_param_multiple_times(self):
        s = QueryString(u'')
        assert (s.add_params([(u'abc', u'123'), (u'def', u'456')]) ==
                s.add_param(u'abc', u'123').add_param(u'def', u'456'))

    def test_add_params_accepts_the_same_args_as_dict(self):
        s = QueryString(u'')
        added = s.add_params({u'abc': u'123'}, foo=u'bar', xyz='456')
        assert added.dict == {u'abc': u'123', u'foo': u'bar', u'xyz': u'456'}
        added2 = s.add_params([(u'abc', u'123')], foo=u'bar', xyz='456')
        assert added2.dict == {u'abc': u'123', u'foo': u'bar', u'xyz': u'456'}

    def test_add_params_accepts_the_same_parameter_name_multiple_times(self):
        s = (QueryString(u'')
             .add_params([(u'abc', u'123'), (u'abc', u'456')]))
        assert s.list == [(u'abc', u'123'), (u'abc', u'456')]

    def test_set_params_is_equivalent_to_calling_set_param_multiple_times(self):
        s = QueryString(u'')
        assert (s.set_params([(u'abc', u'123'), (u'def', u'456')]) ==
                s.set_param(u'abc', u'123').set_param(u'def', u'456'))

    def test_set_params_accepts_the_same_args_as_dict(self):
        s = QueryString(u'')
        added = s.set_params({u'abc': u'123'}, abc='456')
        assert added.dict == {u'abc': u'456'}
        added2 = s.set_params([(u'abc', u'123')], abc=u'456')
        assert added2.dict == {u'abc': u'456'}

    def test_set_params_accepts_the_same_parameter_name_multiple_times(self):
        s = (QueryString(u'')
             .set_params([(u'abc', u'123'), (u'abc', u'456')]))
        assert s.list == [(u'abc', u'456')]

    def test_del_params_accepts_an_iterable_and_removes_all_listed_parameters(self):
        s = QueryString(u'abc=123&def=456&xyz=789')
        assert s.del_params((u'abc', u'xyz')) == u'def=456'
<|endoftext|>


<commit_before>"""
socman is a society membership and event management library.

It tracks members in a sqlite database to enable membership tracking
and verification. It can also track event data such as name and date,
along with event attendance statistics.

The MIT License (MIT)

Copyright (c) 2016 Alexander Thorne

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

from collections import namedtuple
from datetime import date, datetime
import sqlite3

class Name:

    """A person's name."""

    def __init__(self, *names, sep=' '):
        """Create a name from a tuple of strings (passed as variable arguments).

        Arguments:
            names   A list of names which should be strings (or None)
            sep     The separator between names when concatenated to form strings.

        To create a name with just a first name, pass None as the last name:

        >>> my_name = Name('Ted', None)     # Ted is first name
        >>> my_name = Name('Ted')           # Ted is last name

        The full name string will be 'Ted' in both cases above.
        Similarly, to create a name with just a middle name, pass
        None as both the first and the last name:

        >>> my_name = Name(None, 'Ted', None)

        Any number of None names may be passed, as they are ignored when
        building name strings (except for place holding first/last names).
        """
        # None names in list have semantic value, but if list contains only
        # Nones then the Name constructed should be identical to the empty
        # Name constructed by Name()
        if not [name for name in names if name is not None]:
            names = ()

        self.names = list(names)
        self.sep = sep

    def __bool__(self):
        return bool(self.names)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.__dict__ == other.__dict__
        return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __makestr(self, names):
        """Return arguments concatenated together separated by Name.sep.

        Arguments that equal None or are entirely whitespace are omitted.
        """
        return self.sep.join([name for name in names if name and name.strip()])

    def first(self):
        """Return first name as a string."""
        return self.__makestr(self.names[:-1][:1])

    def middle(self):
        """Return middle names concatenated as a string."""
        return self.__makestr(self.names[1:-1])

    def given(self):
        """Return first and any middle names concatenated as a string."""
        return self.__makestr(self.names[:-1])

    def last(self):
        """Return last name as a string."""
        return self.__makestr(self.names[-1:])

    def full(self):
        """Return full name as a string."""
        return self.__makestr(self.names)

Member = namedtuple('Member', 'barcode name college')
Member.__new__.__defaults__ = (None, None)
Member.__doc__ = """
                 A society member.

                 `name` and `college` default to None but `barcode` must be
                 given explicitly.
                 """

class MemberDatabase:

    """Interface to a SQLite3 database of members."""

    def __init__(self, db_file='members.db', safe=True):
        """Create a MemberDatabase.

        Arguments:
            db_file     Filename and path of a SQLite3 database file.
                        Passed directly to sqlite3.Connect().
            safe        Boolean that determines whether non essential
                        operations are committed immediately or not
                        Note that important operations like adding a member
                        are always committed regardless of this setting.
        """

        self.__connection = sqlite3.connect(db_file)
        self.__safe = safe

    def __del__(self):
        self.__connection.commit()  # here, commit regardless of safe
        self.__connection.close()

    # wrapper around sqlite3.Connection.commit():
    # commits if safe is set to True
    # this means users can optionally disable autocommiting for potentially
    # better performance at the cost of reduced data safety on crashes
    def optional_commit(self):
        """Commits changes to database if `safe` is set to `True`"""

        if self.__safe:
            self.__connection.commit()

    def sql_build_name_value_pairs(self, member, sep):
        if not member.name:
            return None, None
        columns = []
        values = ()
        if member.name.first():
            columns += ['firstName=?']
            values += (member.name.first(), )
        if member.name.last():
            columns += ['lastName=?']
            values += (member.name.last(), )
        if not columns:
            return None, None
        return sep.join(columns), values

    def sql_search_barcode_phrase(self, member):
        if not member.barcode:
            return None, None
        return 'barcode=?', (member.barcode, )

    def sql_search_name_phrase(self, member):
        if not member.name:
            return None, None
        return self.sql_build_name_value_pairs(member, ' AND ')

    def sql_search_phrase(self, member):
        phrase, values = self.sql_search_barcode_phrase(member)
        if not phrase:
            phrase, values = self.sql_search_name_phrase(member)
        return phrase, values

    def sql_update_barcode_phrase(self, member):
        if not member.barcode:
            return None, None
        return ('barcode=?', (member.barcode,) )

    def sql_update_name_phrase(self, member):
        return self.sql_build_name_value_pairs(member, ',')

    def sql_search_query(self, member):
        phrase, values = self.sql_search_phrase(member)
        if not phrase:
            return None, None
        return ('SELECT firstName,lastName FROM users WHERE ' + phrase, values)

    def sql_search_barcode_query(self, member):
        phrase, values = self.sql_search_barcode_phrase(member)
        if not phrase:
            return None, None
        return ('SELECT firstName,lastName FROM users WHERE ' + phrase, values)

    def sql_search_name_query(self, member):
        phrase, values = self.sql_search_name_phrase(member)
        if not phrase:
            return None, None
        return ('SELECT firstName,lastName FROM users WHERE ' + phrase, values)

    def sql_update_barcode_query(self, member):
        set_phrase, set_values = self.sql_update_barcode_phrase(member)
        search_phrase, search_values = self.sql_search_name_phrase(member)
        if not set_phrase or not search_phrase:
            return None, None
        return ('UPDATE users SET ' + set_phrase + ' WHERE ' + search_phrase, set_values + search_values)

    def sql_update_name_query(self, member):
        set_phrase, set_values = self.sql_update_name_phrase(member)
        search_phrase, search_values = self.sql_search_barcode_phrase(member)
        if not set_phrase or not search_phrase:
            return None, None
        return ('UPDATE users SET ' + set_phrase + ' WHERE ' + search_phrase, set_values + search_values)

    def sql_update_last_attended_query(self, member):
        search_phrase, search_values = self.sql_search_phrase(member)
        if not search_phrase:
            return None, None
        return ('UPDATE users SET last_attended=? WHERE ' + search_phrase, (date.today(),) + search_values)

    def get_member(self, barcode=None, *, member=None, update_timestamp=True, autofix=False):
        c = self.__connection.cursor()

        if not member:
            if not barcode:
                return None
            member = Member(barcode)

        # first try to find member by barcode, if available
        query, values = self.sql_search_barcode_query(member)
        if query:
            c.execute(query, values)
            users = c.fetchall()
            if users:
                # if necessary update last_attended date
                if update_timestamp:
                    ts_query, ts_values = self.sql_update_last_attended_query(member)
                    if ts_query:
                        c.execute(ts_query, ts_values)

                # if autofix is set and both names are provided, correct names
                if autofix:
                    af_query, af_values = self.sql_update_name_query(member)
                    if af_query:
                        c.execute(af_query, af_values)

                # in case we updated timestamp or names, commit
                self.optional_commit()

                # TODO dedupe if necessary
                return users[0]

        # barcode lookup failed; now try finding by name
        query, values = self.sql_search_name_query(member)
        if not query:
            return None

        c.execute(query, values)
        users = c.fetchall()

        if not users:
            return None  # still nothing found

        # found them so update last_attended if update_timestamp is set
        if update_timestamp:
            # todo find better way to choose search phrase (barcode/name)
            temp_member = member._replace(barcode = None)
            ts_query, ts_values = self.sql_update_last_attended_query(temp_member)
            if ts_query:
                c.execute(ts_query, ts_values)
        # and barcode if autofix is set
        if autofix and member.barcode:
            af_query, af_values = self.sql_update_barcode_query(member)
            c.execute(af_query, af_values)
            self.optional_commit()

        # TODO dedupe if necessary
        return users[0]

    def sql_add_query(self, member):
        return 'INSERT INTO users (barcode, firstName, lastName, college, datejoined, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)', (member.barcode, member.name.first(), member.name.last(), member.college, date.today(), datetime.utcnow(), datetime.utcnow())

    def add_member(self, member):
        if not member:
            return False

        if self.get_member(member=member, update_timestamp=False, autofix=True):
            return False

        # if member does not exist, add them
        cursor = self.__connection.cursor()
        cursor.execute(*self.sql_add_query(member))
        self.__connection.commit()  # direct commit here: don't want to lose new member data

        return True
<commit_msg>Better separate license from rest of docstring<commit_after>"""
socman is a society membership and event management library.

It tracks members in a sqlite database to enable membership tracking
and verification. It can also track event data such as name and date,
along with event attendance statistics.

The MIT License (MIT)

Copyright (c) 2016 Alexander Thorne

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

from collections import namedtuple
from datetime import date, datetime
import sqlite3

class Name:

    """A person's name."""

    def __init__(self, *names, sep=' '):
        """Create a name from a tuple of strings (passed as variable arguments).

        Arguments:
            names   A list of names which should be strings (or None)
            sep     The separator between names when concatenated to form strings.

        To create a name with just a first name, pass None as the last name:

        >>> my_name = Name('Ted', None)     # Ted is first name
        >>> my_name = Name('Ted')           # Ted is last name

        The full name string will be 'Ted' in both cases above.
        Similarly, to create a name with just a middle name, pass
        None as both the first and the last name:

        >>> my_name = Name(None, 'Ted', None)

        Any number of None names may be passed, as they are ignored when
        building name strings (except for place holding first/last names).
        """
        # None names in list have semantic value, but if list contains only
        # Nones then the Name constructed should be identical to the empty
        # Name constructed by Name()
        if not [name for name in names if name is not None]:
            names = ()

        self.names = list(names)
        self.sep = sep

    def __bool__(self):
        return bool(self.names)

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.__dict__ == other.__dict__
        return False

    def __ne__(self, other):
        return not self.__eq__(other)

    def __makestr(self, names):
        """Return arguments concatenated together separated by Name.sep.

        Arguments that equal None or are entirely whitespace are omitted.
        """
        return self.sep.join([name for name in names if name and name.strip()])

    def first(self):
        """Return first name as a string."""
        return self.__makestr(self.names[:-1][:1])

    def middle(self):
        """Return middle names concatenated as a string."""
        return self.__makestr(self.names[1:-1])

    def given(self):
        """Return first and any middle names concatenated as a string."""
        return self.__makestr(self.names[:-1])

    def last(self):
        """Return last name as a string."""
        return self.__makestr(self.names[-1:])

    def full(self):
        """Return full name as a string."""
        return self.__makestr(self.names)

Member = namedtuple('Member', 'barcode name college')
Member.__new__.__defaults__ = (None, None)
Member.__doc__ = """
                 A society member.

                 `name` and `college` default to None but `barcode` must be
                 given explicitly.
                 """

class MemberDatabase:

    """Interface to a SQLite3 database of members."""

    def __init__(self, db_file='members.db', safe=True):
        """Create a MemberDatabase.

        Arguments:
            db_file     Filename and path of a SQLite3 database file.
                        Passed directly to sqlite3.Connect().
            safe        Boolean that determines whether non essential
                        operations are committed immediately or not
                        Note that important operations like adding a member
                        are always committed regardless of this setting.
        """

        self.__connection = sqlite3.connect(db_file)
        self.__safe = safe

    def __del__(self):
        self.__connection.commit()  # here, commit regardless of safe
        self.__connection.close()

    # wrapper around sqlite3.Connection.commit():
    # commits if safe is set to True
    # this means users can optionally disable autocommiting for potentially
    # better performance at the cost of reduced data safety on crashes
    def optional_commit(self):
        """Commits changes to database if `safe` is set to `True`"""

        if self.__safe:
            self.__connection.commit()

    def sql_build_name_value_pairs(self, member, sep):
        if not member.name:
            return None, None
        columns = []
        values = ()
        if member.name.first():
            columns += ['firstName=?']
            values += (member.name.first(), )
        if member.name.last():
            columns += ['lastName=?']
            values += (member.name.last(), )
        if not columns:
            return None, None
        return sep.join(columns), values

    def sql_search_barcode_phrase(self, member):
        if not member.barcode:
            return None, None
        return 'barcode=?', (member.barcode, )

    def sql_search_name_phrase(self, member):
        if not member.name:
            return None, None
        return self.sql_build_name_value_pairs(member, ' AND ')

    def sql_search_phrase(self, member):
        phrase, values = self.sql_search_barcode_phrase(member)
        if not phrase:
            phrase, values = self.sql_search_name_phrase(member)
        return phrase, values

    def sql_update_barcode_phrase(self, member):
        if not member.barcode:
            return None, None
        return ('barcode=?', (member.barcode,) )

    def sql_update_name_phrase(self, member):
        return self.sql_build_name_value_pairs(member, ',')

    def sql_search_query(self, member):
        phrase, values = self.sql_search_phrase(member)
        if not phrase:
            return None, None
        return ('SELECT firstName,lastName FROM users WHERE ' + phrase, values)

    def sql_search_barcode_query(self, member):
        phrase, values = self.sql_search_barcode_phrase(member)
        if not phrase:
            return None, None
        return ('SELECT firstName,lastName FROM users WHERE ' + phrase, values)

    def sql_search_name_query(self, member):
        phrase, values = self.sql_search_name_phrase(member)
        if not phrase:
            return None, None
        return ('SELECT firstName,lastName FROM users WHERE ' + phrase, values)

    def sql_update_barcode_query(self, member):
        set_phrase, set_values = self.sql_update_barcode_phrase(member)
        search_phrase, search_values = self.sql_search_name_phrase(member)
        if not set_phrase or not search_phrase:
            return None, None
        return ('UPDATE users SET ' + set_phrase + ' WHERE ' + search_phrase, set_values + search_values)

    def sql_update_name_query(self, member):
        set_phrase, set_values = self.sql_update_name_phrase(member)
        search_phrase, search_values = self.sql_search_barcode_phrase(member)
        if not set_phrase or not search_phrase:
            return None, None
        return ('UPDATE users SET ' + set_phrase + ' WHERE ' + search_phrase, set_values + search_values)

    def sql_update_last_attended_query(self, member):
        search_phrase, search_values = self.sql_search_phrase(member)
        if not search_phrase:
            return None, None
        return ('UPDATE users SET last_attended=? WHERE ' + search_phrase, (date.today(),) + search_values)

    def get_member(self, barcode=None, *, member=None, update_timestamp=True, autofix=False):
        c = self.__connection.cursor()

        if not member:
            if not barcode:
                return None
            member = Member(barcode)

        # first try to find member by barcode, if available
        query, values = self.sql_search_barcode_query(member)
        if query:
            c.execute(query, values)
            users = c.fetchall()
            if users:
                # if necessary update last_attended date
                if update_timestamp:
                    ts_query, ts_values = self.sql_update_last_attended_query(member)
                    if ts_query:
                        c.execute(ts_query, ts_values)

                # if autofix is set and both names are provided, correct names
                if autofix:
                    af_query, af_values = self.sql_update_name_query(member)
                    if af_query:
                        c.execute(af_query, af_values)

                # in case we updated timestamp or names, commit
                self.optional_commit()

                # TODO dedupe if necessary
                return users[0]

        # barcode lookup failed; now try finding by name
        query, values = self.sql_search_name_query(member)
        if not query:
            return None

        c.execute(query, values)
        users = c.fetchall()

        if not users:
            return None  # still nothing found

        # found them so update last_attended if update_timestamp is set
        if update_timestamp:
            # todo find better way to choose search phrase (barcode/name)
            temp_member = member._replace(barcode = None)
            ts_query, ts_values = self.sql_update_last_attended_query(temp_member)
            if ts_query:
                c.execute(ts_query, ts_values)
        # and barcode if autofix is set
        if autofix and member.barcode:
            af_query, af_values = self.sql_update_barcode_query(member)
            c.execute(af_query, af_values)
            self.optional_commit()

        # TODO dedupe if necessary
        return users[0]

    def sql_add_query(self, member):
        return 'INSERT INTO users (barcode, firstName, lastName, college, datejoined, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?)', (member.barcode, member.name.first(), member.name.last(), member.college, date.today(), datetime.utcnow(), datetime.utcnow())

    def add_member(self, member):
        if not member:
            return False

        if self.get_member(member=member, update_timestamp=False, autofix=True):
            return False

        # if member does not exist, add them
        cursor = self.__connection.cursor()
        cursor.execute(*self.sql_add_query(member))
        self.__connection.commit()  # direct commit here: don't want to lose new member data

        return True
<|endoftext|>


<commit_before><commit_msg>Added back in the replace statement<commit_after><|endoftext|>


<commit_before># Ben Eggers <ben.eggers36@gmail.com>
#
# A simple, gradient-descent-based linear system solver. The idea is as follows:
#
# We are given a matrix A and a vector b, and want to find x such that Ax = b.
#
# We turn this into an optimization problem, with our objective function f(x) =
# the A-norm of the error (Ax - b). We use an iterative method. At each step,
# we find the gradient at our current guess (initialized to 0), then take the
# optimal step size (computable because matrices are awesome) in the direction
# of the gradient. We repeat until the gradient = 0, which is when our guess
# will be the solution to the system.
#
# I only want to deal with SDD systems here, so the matrix input will be in
# graph form, with the matrix defined to be the Laplacian of the input graph.

import numpy as np
import time

# Get the gradient of A at x
def grad(A, b, x):
    return -2 * rhe(A, b, x)

# Error on the right-hand side. b - Ax
def rhe(A, b, x):
    return b - A.dot(x)

def get_matrix():
    return np.array([1, -1, 0,     -1, 2, -1,     0, -1, 2 ]).reshape((3, 3))

def get_target(length):
    print "Please input", length, "numbers for the target vector"
    ret = []
    while len(ret) < length:
        ret.append(int(raw_input("Next number? ")))
    return np.array(ret)

def report(A, b, x):
    print "x =", x
    print "Error norm:", np.linalg.norm(rhe(A, b, x))

# ititial conditions
A = get_matrix()
b = get_target(A.shape[0])

x = np.array([0 for i in range(b.shape[0])])

# do the actual descent
g = grad(A, b, x)
# Ideally we'd go until the norm equals 0, but numerical analysis is a thing
while np.linalg.norm(g) > 0.00005:
    r = rhe(A, b, x)
    x = x + ((1. * r.dot(r)) / (r.dot(A).dot(r))) * r
    g = grad(A, b, x)

# Now we can round, since we're pretty much guaranteed to be within .5 of
# the correct answer
x = np.array([round(n) for n in x])

report(A, b, x)
<commit_msg>finished<commit_after># Ben Eggers <ben.eggers36@gmail.com>
#
# A simple, gradient-descent-based linear system solver. The idea is as follows:
#
# We are given a matrix A and a vector b, and want to find x such that Ax = b.
#
# We turn this into an optimization problem, with our objective function f(x) =
# the A-norm of the error (Ax - b). We use an iterative method. At each step,
# we find the gradient at our current guess (initialized to 0), then take the
# optimal step size (computable because matrices are awesome) in the direction
# of the gradient. We repeat until the gradient = 0, which is when our guess
# will be the solution to the system.
#
# I only want to deal with SDD systems here, so the matrix input will be in
# graph form, with the matrix defined to be the Laplacian of the input graph.

import numpy as np
import time

# Get the gradient of A at x
def grad(A, b, x):
    return -2 * rhe(A, b, x)

# Error on the right-hand side. b - Ax
def rhe(A, b, x):
    return b - A.dot(x)

def get_matrix():
    ec = int(raw_input("How many edges on input graph? "))
    edges = []
    for i in xrange(ec):
        inp = raw_input("Next edge(of the form '<node_#>, <node_#>'): ")
        edges.append((int(inp.split(", ")[0]), int(inp.split(", ")[1])))

    m = int(max([max(e[0], e[1]) for e in edges]))

    ret = np.zeros((m, m), dtype = np.int)

    for e in edges:
        ret[e[0] - 1][e[1] - 1] = ret[e[1] - 1][e[0] - 1] = -1
        # These should be "+= 1" for the true Laplacian, but we need the
        # matrix to be non-singular. So...
        ret[e[0] - 1][e[0] - 1] += 2
        ret[e[1] - 1][e[1] - 1] += 2

    print ret
    return ret

def get_target(length):
    print "Please input", length, "numbers for the target vector"
    ret = []
    while len(ret) < length:
        ret.append(int(raw_input("Next number? ")))
    return np.array(ret)

def report(A, b, x):
    print "x =", x
    print "Error norm:", np.linalg.norm(rhe(A, b, x))

# ititial conditions
A = get_matrix()
b = get_target(A.shape[0])

x = np.array([0 for i in range(b.shape[0])])

# do the actual descent
g = grad(A, b, x)
# Ideally we'd go until the norm equals 0, but numerical analysis is a thing
while np.linalg.norm(g) > 0.000005:
    r = rhe(A, b, x)
    x = x + ((1. * r.dot(r)) / (r.dot(A).dot(r))) * r
    g = grad(A, b, x)

# Now we can round, since we're pretty much guaranteed to be within .5 of
# the correct answer
x = np.array([round(n) for n in x])

report(A, b, x)
<|endoftext|>