<commit_before># 1. Add the directory "lib/rfid" to the sys.path array to make it available for Python
# 2. Create a variable with the path for the userDB database
import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), "../lib/MFRC522-python"))
dbDir = os.path.join(os.path.dirname(__file__), "../resources/main.db")

import time, sqlite3

# Library to allow me to control the GPIO pins on the RPi
try:
    import RPi.GPIO as GPIO
except RuntimeError:
    print("Error importing RPi.GPIO")

GPIO.setwarnings(False)

# Library for the RC522 RFID reader
# https://github.com/mxgxw/MFRC522-python/
try:
    import MFRC522
except RuntimeError:
    print("Error importing MFRC522")

def scan():

    # Create an object of MFRC522 which will allow me to scan for cards, read the UID of the card and authenticate it.
    reader = MFRC522.MFRC522()

    # Loop to keep scanning for tags/cards
    while True:

        '''
       - Send a request to the reader to scan for any present RFID cards/tags.
       - 'status' is used to verify the status of the reader (if the user has tapped a card/tag) .
       - 'TagType' is used to validate that the present card/tag is a MIFARE card.
       - This variable doesn't have to be used as it would only be used to warn the user, however because there is no console for the user there is not much need for this.
       '''
        # Check if a card is present
        (status,TagType) = reader.MFRC522_Request(reader.PICC_REQIDL)

        # If no card is found then it will continue the loop
	    # If a card is present the program will run this if statement
        if status == reader.MI_OK:

            # Read the UID of the card, ready to query the database.
            (status,uid) = reader.MFRC522_Anticoll()

    	    # Take the values in 'uid' list and concatenate into a string for database query
    	    uidStr = ""
    	    for number in uid:
    	        uidStr = uidStr + str(number)
    	    print int(uidStr)

            if uidStr == "7660887599":
                time.sleep(2)
                masterCard()
            else:
                isValid = queryUID(uidStr)

        	if isValid == True:
        	    # Change the state of the door to it's opposite state
        	    operateDoor()

        	    state = GPIO.input(3)
        	    if state == 0:
    	    	    # Delay to scan after 2 seconds to see if the RFID signal is still present
                        time.sleep(2)

        	    	# Take the current time (seconds since Jan 1st 1970) and store as a temp value
        	    	tempTimeVal = time.time()

        	    	# Set the default mode to lock automatically after the door is unlocked
        	    	changeState = True

        	    	# Create a loop which runs for 0.5 seconds to scan for RFID signals
        	    	while (time.time() - tempTimeVal) < 0.5:
        	    	    (status,TagType) = reader.MFRC522_Request(reader.PICC_REQIDL)

        		    # Check if card is still present
        		    if status == reader.MI_OK:

        		    	# Set the door to stay unlocked once originally unlocked
        		    	changeState = False

        	    	# Automatically lock the door if changeState is it's default value (True)
        	    	if changeState == True:
        		    operateDoor(uidStr)

        # Delay for 3 seconds before the script can be run again
	    time.sleep(2)

def queryUID(id):
    # Connect to the user database
    conn = sqlite3.connect(dbDir)
    curs = conn.cursor()
    returnValue = curs.execute("SELECT ID FROM USERS WHERE ID = "+id).fetchone()
    conn.close()
    if returnValue is not None:
        return True
    else:
        return False

def masterCard():
    print "Master card activated."
    reader = MFRC522.MFRC522()
    continueLoop = True
    while continueLoop == True:
        (status,TagType) = reader.MFRC522_Request(reader.PICC_REQIDL)
        if status == reader.MI_OK:
            (status,uid) = reader.MFRC522_Anticoll()
            uidStr = ""
    	    for number in uid:
    	        uidStr = uidStr + str(number)
    	    print int(uidStr)
            if uidStr == "7660887599":
                continueLoop = False
            else:
                # Connect to the user database
                conn = sqlite3.connect(dbDir)
                curs = conn.cursor()
                returnValue = curs.execute("SELECT id from USERS where id = "+uidStr).fetchone()
                print returnValue
                conn.close()
                if returnValue is not None:
                    print "Remove."
                    rmUID(uidStr)
                    continueLoop = False
                else:
                    print "Add."
                    addUID(uidStr)
                    continueLoop = False
    print "Master card deactivated."

def addUID(id):
    conn = sqlite3.connect(dbDir)
    curs = conn.cursor()
    curs.execute("INSERT INTO USERS (ID) VALUES ("+id+")")
    conn.commit()
    conn.close()

def rmUID(id):
    conn = sqlite3.connect(dbDir)
    curs = conn.cursor()
    curs.execute("DELETE FROM USERS WHERE id = "+id)
    conn.commit()
    conn.close()

def operateDoor(id):
    GPIO.setup(3, GPIO.OUT)
    state = GPIO.input(3)
    if state == 0:
        status = False
	    GPIO.output(3, GPIO.HIGH)
        logTime(opened, id)
    elif state == 1:
        status = True
        GPIO.output(3, GPIO.LOW)
        logTime(opened, id)

def logTime(status, id):
    currentTime = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
    conn = sqlite3.connect(dbDir)
    curs = conn.cursor()
    print status + id + currentTime
    curs.execute("INSERT INTO LOGS (UID, TIME, STATUS) VALUES ("+id+", "+currentTime+", "+status+")")

scan()
<commit_msg>Linked 2 databases into 1<commit_after># 1. Add the directory "lib/rfid" to the sys.path array to make it available for Python
# 2. Create a variable with the path for the userDB database
import os, sys
sys.path.append(os.path.join(os.path.dirname(__file__), "../lib/MFRC522-python"))
dbDir = os.path.join(os.path.dirname(__file__), "../resources/main.db")

import time, sqlite3

# Library to allow me to control the GPIO pins on the RPi
try:
    import RPi.GPIO as GPIO
except RuntimeError:
    print("Error importing RPi.GPIO")

GPIO.setwarnings(False)

# Library for the RC522 RFID reader
# https://github.com/mxgxw/MFRC522-python/
try:
    import MFRC522
except RuntimeError:
    print("Error importing MFRC522")

def scan():

    # Create an object of MFRC522 which will allow me to scan for cards, read the UID of the card and authenticate it.
    reader = MFRC522.MFRC522()

    # Loop to keep scanning for tags/cards
    while True:

        '''
       - Send a request to the reader to scan for any present RFID cards/tags.
       - 'status' is used to verify the status of the reader (if the user has tapped a card/tag) .
       - 'TagType' is used to validate that the present card/tag is a MIFARE card.
       - This variable doesn't have to be used as it would only be used to warn the user, however because there is no console for the user there is not much need for this.
       '''
        # Check if a card is present
        (status,TagType) = reader.MFRC522_Request(reader.PICC_REQIDL)

        # If no card is found then it will continue the loop
	    # If a card is present the program will run this if statement
        if status == reader.MI_OK:

            # Read the UID of the card, ready to query the database.
            (status,uid) = reader.MFRC522_Anticoll()

    	    # Take the values in 'uid' list and concatenate into a string for database query
    	    uidStr = ""
    	    for number in uid:
    	        uidStr = uidStr + str(number)
    	    print int(uidStr)

            if uidStr == "7660887599":
                time.sleep(2)
                masterCard()
            else:
                isValid = queryUID(uidStr)

        	if isValid == True:
        	    # Change the state of the door to it's opposite state
        	    operateDoor(uidStr)

        	    state = GPIO.input(3)
        	    if state == 0:
    	    	    # Delay to scan after 2 seconds to see if the RFID signal is still present
                        time.sleep(2)

        	    	# Take the current time (seconds since Jan 1st 1970) and store as a temp value
        	    	tempTimeVal = time.time()

        	    	# Set the default mode to lock automatically after the door is unlocked
        	    	changeState = True

        	    	# Create a loop which runs for 0.5 seconds to scan for RFID signals
        	    	while (time.time() - tempTimeVal) < 0.5:
        	    	    (status,TagType) = reader.MFRC522_Request(reader.PICC_REQIDL)

        		    # Check if card is still present
        		    if status == reader.MI_OK:

        		    	# Set the door to stay unlocked once originally unlocked
        		    	changeState = False

        	    	# Automatically lock the door if changeState is it's default value (True)
        	    	if changeState == True:
        		    operateDoor(uidStr)

        # Delay for 3 seconds before the script can be run again
	    time.sleep(2)

def queryUID(id):
    # Connect to the user database
    conn = sqlite3.connect(dbDir)
    curs = conn.cursor()
    returnValue = curs.execute("SELECT ID FROM USERS WHERE ID = "+id).fetchone()
    conn.close()
    if returnValue is not None:
        return True
    else:
        return False

def masterCard():
    print "Master card activated."
    reader = MFRC522.MFRC522()
    continueLoop = True
    while continueLoop == True:
        (status,TagType) = reader.MFRC522_Request(reader.PICC_REQIDL)
        if status == reader.MI_OK:
            (status,uid) = reader.MFRC522_Anticoll()
            uidStr = ""
    	    for number in uid:
    	        uidStr = uidStr + str(number)
    	    print int(uidStr)
            if uidStr == "7660887599":
                continueLoop = False
            else:
                # Connect to the user database
                conn = sqlite3.connect(dbDir)
                curs = conn.cursor()
                returnValue = curs.execute("SELECT id from USERS where id = "+uidStr).fetchone()
                print returnValue
                conn.close()
                if returnValue is not None:
                    print "Remove."
                    rmUID(uidStr)
                    continueLoop = False
                else:
                    print "Add."
                    addUID(uidStr)
                    continueLoop = False
    print "Master card deactivated."

def addUID(id):
    conn = sqlite3.connect(dbDir)
    curs = conn.cursor()
    curs.execute("INSERT INTO USERS (ID) VALUES ("+id+")")
    conn.commit()
    conn.close()

def rmUID(id):
    conn = sqlite3.connect(dbDir)
    curs = conn.cursor()
    curs.execute("DELETE FROM USERS WHERE id = "+id)
    conn.commit()
    conn.close()

def operateDoor(id):
    GPIO.setup(3, GPIO.OUT)
    state = GPIO.input(3)
    if state == 0:
        status = False
	    GPIO.output(3, GPIO.HIGH)
        logTime(opened, id)
    elif state == 1:
        status = True
        GPIO.output(3, GPIO.LOW)
        logTime(opened, id)

def logTime(status, id):
    currentTime = strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
    conn = sqlite3.connect(dbDir)
    curs = conn.cursor()
    print status + id + currentTime
    curs.execute("INSERT INTO LOGS (UID, TIME, STATUS) VALUES ("+id+", "+currentTime+", "+status+")")

scan()
<|endoftext|>


<commit_before>import argparse
import subprocess
import sys
from typing import List, Optional

from dtags import style
from dtags.commons import (
    dtags_command,
    fix_color_for_windows,
    get_argparser,
    normalize_dir,
    normalize_tag,
    reverse_map,
)
from dtags.files import load_config_file

USAGE = "run DEST [DEST ...] -c ..."
DESCRIPTION = f"""
Execute a command in one or more directories.

Target directories are iterated in alphabetical order.
Paths take precedence over tags on name collisions.
The command is run only once per directory in subprocesses.

examples:

  # run "git status" in all directories tagged "work"
  {style.command("run work -c git status")}

  # run "git status" in directories ~/foo and ~/bar
  {style.command("run ~/foo ~/bar -c git status")}

  # run "git status" in directories tagged "work" and in ~/foo
  {style.command("run work ~/foo -c git status")}
"""

@dtags_command
def execute(args: Optional[List[str]] = None) -> None:
    parser = get_argparser(prog="run", desc=DESCRIPTION, usage=USAGE)
    parser.add_argument(
        "destinations",
        metavar="DEST",
        nargs="+",
        help="directory path or tag",
    )
    parser.add_argument(
        "-c",
        "--cmd",
        dest="command",
        nargs=argparse.REMAINDER,
        required=True,
        help="command to execute",
    )
    parsed_args = parser.parse_args(sys.argv[1:] if args is None else args)

    if not parsed_args.command:
        parser.error("the following arguments are required: -c/--cmd")
    else:
        run_command(parsed_args.destinations, parsed_args.command)

def run_command(destinations: List[str], command: List[str]) -> None:
    config = load_config_file()
    tag_config = config["tags"]

    tag_to_dirpaths = reverse_map(tag_config)
    dirpaths = set()

    for dest in destinations:
        dirpath = normalize_dir(dest)
        if dirpath is not None:
            dirpaths.add(dirpath)
        else:
            tag = normalize_tag(dest)
            if tag in tag_to_dirpaths:
                dirpaths.update(tag_to_dirpaths[tag])

    return_code = 0
    for dirpath in sorted(dirpaths):
        tags = tag_config.get(dirpath, set())

        fix_color_for_windows()
        print(f"\n{style.mapping(dirpath, tags)}:")
        try:
            process = subprocess.run(
                command,
                cwd=dirpath,
                stderr=subprocess.STDOUT,
            )
        except FileNotFoundError:
            print(f"Invalid command: {command[0]}", file=sys.stderr)
        except NotADirectoryError:  # pragma no cover
            print(f"Not a directory: {dirpath.as_posix()}", file=sys.stderr)
        else:
            if process.returncode != 0:
                return_code = 1

    sys.exit(return_code)
<commit_msg>Skip missing directories in run command<commit_after>import argparse
import subprocess
import sys
from typing import List, Optional

from dtags import style
from dtags.commons import (
    dtags_command,
    fix_color_for_windows,
    get_argparser,
    normalize_dir,
    normalize_tag,
    reverse_map,
)
from dtags.files import load_config_file

USAGE = "run DEST [DEST ...] -c ..."
DESCRIPTION = f"""
Execute a command in one or more directories.

Target directories are iterated in alphabetical order.
Paths take precedence over tags on name collisions.
The command is run only once per directory in subprocesses.

examples:

  # run "git status" in all directories tagged "work"
  {style.command("run work -c git status")}

  # run "git status" in directories ~/foo and ~/bar
  {style.command("run ~/foo ~/bar -c git status")}

  # run "git status" in directories tagged "work" and in ~/foo
  {style.command("run work ~/foo -c git status")}
"""

@dtags_command
def execute(args: Optional[List[str]] = None) -> None:
    parser = get_argparser(prog="run", desc=DESCRIPTION, usage=USAGE)
    parser.add_argument(
        "destinations",
        metavar="DEST",
        nargs="+",
        help="directory path or tag",
    )
    parser.add_argument(
        "-c",
        "--cmd",
        dest="command",
        nargs=argparse.REMAINDER,
        required=True,
        help="command to execute",
    )
    parsed_args = parser.parse_args(sys.argv[1:] if args is None else args)

    if not parsed_args.command:
        parser.error("the following arguments are required: -c/--cmd")
    else:
        run_command(parsed_args.destinations, parsed_args.command)

def run_command(destinations: List[str], command: List[str]) -> None:
    config = load_config_file()
    tag_config = config["tags"]

    tag_to_dirpaths = reverse_map(tag_config)
    dirpaths = set()

    for dest in destinations:
        dirpath = normalize_dir(dest)
        if dirpath is not None:
            dirpaths.add(dirpath)
        else:
            tag = normalize_tag(dest)
            if tag in tag_to_dirpaths:
                for dirpath in tag_to_dirpaths[tag]:
                    if dirpath.is_dir():
                        dirpaths.add(dirpath)

    return_code = 0
    for dirpath in sorted(dirpaths):
        tags = tag_config.get(dirpath, set())

        fix_color_for_windows()
        print(f"\n{style.mapping(dirpath, tags)}:")
        try:
            process = subprocess.run(
                command,
                cwd=dirpath,
                stderr=subprocess.STDOUT,
            )
        except FileNotFoundError:
            print(f"Invalid command: {command[0]}", file=sys.stderr)
        except NotADirectoryError:  # pragma no cover
            print(f"Not a directory: {dirpath.as_posix()}", file=sys.stderr)
        else:
            if process.returncode != 0:
                return_code = 1

    sys.exit(return_code)
<|endoftext|>


<commit_before>"""
Extension of PyWavelets to complex wavelets suitable for the Dual-Tree Complex Wavelet Transform.

Author : Laurent P. René de Cotret
"""
import numpy as n
from os.path import join, dirname
from pywt import Wavelet, wavelist
from warnings import warn

# lru_cache only exists as of Python 3.2
try:
    from functools import lru_cache
except ImportError:
    warn('functools.lru_cache could not be found. Performance will be affected.', ImportWarning)
    def lru_cache(func, *args, **kwargs): return func

__all__ = ['dualtree_wavelet', 'dualtree_first_stage']

DATADIR = join(dirname(__file__), 'data')
ALL_QSHIFT = ('qshift1', 'qshift2', 'qshift3', 'qshift4')
ALL_COMPLEX_WAV = ('kingsbury99',) + ALL_QSHIFT
ALL_FIRST_STAGE = ('kingsbury99_fs',) + tuple([wav for wav in wavelist() if wav != 'dmey'])

@lru_cache(maxsize = len(ALL_COMPLEX_WAV))
def dualtree_wavelet(name):
    """
    Returns a complex wavelet suitable for dual-tree cwt from a name.

    Parameters
    ----------
    name : str, {'qshift1', 'qshift2', 'qshift3', 'qshift4', 'kingsbury99'}
        Valid arguments can be found in dualtree.ALL_COMPLEX_WAV

    Returns
    -------
    real, imag : pywt.Wavelet objects.

    Raises
    ------
    ValueError
        If illegal wavelet name.
    """
    if name not in ALL_COMPLEX_WAV:
        raise ValueError('{} is not associated with any implemented complex wavelet. Possible choices are {}'.format(name, ALL_COMPLEX_WAV))

    # This function is simply for now
    if name == 'kingsbury99':
        return kingsbury99()

    return _qshift(name)

@lru_cache(maxsize = len(ALL_FIRST_STAGE))
def dualtree_first_stage(wavelet = 'kingsbury99_fs'):
    """
    Returns two wavelets to be used in the dual-tree complex wavelet transform, at the first stage.

    Parameters
    ----------
    wavelet : str or Wavelet
        Wavelet to be shifted for first-stage use. Valid arguments can be found in dualtree.ALL_FIRST_STAGE

    Return
    ------
    wav1, wav2 : Wavelet objects

    Raises
    ------
    ValueError
        If invalid first stage wavelet.
    """
    # Special case, preshifted
    if wavelet == 'kingsbury99_fs':
        return kingsbury99_fs()

    if not isinstance(wavelet, Wavelet):
        wavelet = Wavelet(wavelet)

    if wavelet.name not in ALL_FIRST_STAGE:
        raise ValueError('{} is an invalid first stage wavelet.'.format(wavelet.name))

    # extend filter bank with zeros
    filter_bank = [n.array(f, copy = True) for f in wavelet.filter_bank]
    for filt in filter_bank:
        extended = n.zeros( shape = (filt.shape[0] + 2,), dtype = n.float)
        extended[1:-1] = filt
        filt = extended

    # Shift deconstruction filters to one side, and reconstruction
    # to the other side
    shifted_fb = [n.array(f, copy = True) for f in wavelet.filter_bank]
    for filt in shifted_fb[::2]:    #Deconstruction filters
        filt = n.roll(filt, 1)
    for filt in shifted_fb[2::]:    # Reconstruction filters
        filt = n.roll(filt, -1)

    return Wavelet(name = wavelet.name, filter_bank = filter_bank), Wavelet(name = wavelet.name, filter_bank = shifted_fb)

@lru_cache(maxsize = len(ALL_QSHIFT))
def _qshift(name):
    """
    Returns a complex qshift wavelet by name.

    Parameters
    ----------
    name : str
        Wavelet to use. Valid arguments can be found in dualtree.ALL_QSHIFT

    Returns
    -------
    wav1, wav2 : pywt.Wavelet objects
        real and imaginary wavelet

    Raises
    ------
    ValueError
        If illegal wavelet family name.
    """
    filters = ('h0a', 'h0b', 'g0a', 'g0b', 'h1a', 'h1b', 'g1a', 'g1b')

    filename = join(DATADIR, name + '.npz')
    with n.load(filename) as mat:
        try:
            (dec_real_low, dec_imag_low, rec_real_low, rec_imag_low,
             dec_real_high, dec_imag_high, rec_real_high, rec_imag_high) = tuple([mat[k].flatten() for k in filters])
        except KeyError:
            raise ValueError('Wavelet does not define ({0}) coefficients'.format(', '.join(filters)))

    real_filter_bank = [dec_real_low, dec_real_high, rec_real_low, rec_real_high]
    imag_filter_bank = [dec_imag_low, dec_imag_high, rec_imag_low, rec_imag_high]

    return Wavelet(name = 'real:' + name, filter_bank = real_filter_bank), Wavelet(name = 'imag:' + name, filter_bank = imag_filter_bank)

#############################################################################################
#                           EXAMPLE COMPLEX WAVELETS FROM
#               http://eeweb.poly.edu/iselesni/WaveletSoftware/dt1D.html
#############################################################################################
@lru_cache(maxsize = 1)
def kingsbury99_fs():
    """
    Returns a first-stage complex wavelet as published in [1].

    References
    ----------
    [1] Kingsbury, N. 'Image Processing with Complex Wavelets'. Philocophical Transactions of the Royal Society A pp. 2543-2560, September 1999
    """
    real_dec_lo = n.array([0, -0.08838834764832, 0.08838834764832, 0.69587998903400,0.69587998903400, 0.08838834764832, -0.08838834764832, 0.01122679215254, 0.01122679215254, 0])
    real_dec_hi = n.array([0, -0.01122679215254, 0.01122679215254, 0.08838834764832, 0.08838834764832, -0.69587998903400, 0.69587998903400, -0.08838834764832, -0.08838834764832, 0])
    real_rec_lo, real_rec_hi = real_dec_lo[::-1], real_dec_hi[::-1]

    imag_dec_lo = n.array([0.01122679215254, 0.01122679215254, -0.08838834764832, 0.08838834764832, 0.69587998903400, 0.69587998903400, 0.08838834764832, -0.08838834764832, 0, 0])
    imag_dec_hi = n.array([0, 0, -0.08838834764832, -0.08838834764832, 0.69587998903400, -0.69587998903400, 0.08838834764832, 0.08838834764832, 0.01122679215254, -0.01122679215254])
    imag_rec_lo, imag_rec_hi = imag_dec_lo[::-1], imag_dec_hi[::-1]

    real_fb = [real_dec_lo, real_dec_hi, real_rec_lo, real_rec_hi]
    imag_fb = [imag_dec_lo, imag_dec_hi, imag_rec_lo, imag_rec_hi]

    return Wavelet(name = 'real:', filter_bank = real_fb), Wavelet(name = 'imag:', filter_bank = imag_fb)

@lru_cache(maxsize = 1)
def kingsbury99():
    """
    Returns a late-stage complex wavelet as published in [1].

    References
    ----------
    [1] Kingsbury, N. 'Image Processing with Complex Wavelets'. Philocophical Transactions of the Royal Society A pp. 2543-2560, September 1999
    """
    real_dec_lo = n.array([ 0.03516384000000, 0, -0.08832942000000, 0.23389032000000, 0.76027237000000, 0.58751830000000, 0, -0.11430184000000, 0, 0])
    real_dec_hi = n.array([0, 0, -0.11430184000000, 0, 0.58751830000000, -0.76027237000000, 0.23389032000000, 0.08832942000000, 0, -0.03516384000000])
    real_rec_lo, real_rec_hi = real_dec_lo[::-1], real_dec_hi[::-1]

    imag_dec_lo = n.array([ 0, 0, -0.11430184000000, 0, 0.58751830000000, 0.76027237000000, 0.23389032000000, -0.08832942000000, 0, 0.03516384000000])
    imag_dec_hi = n.array([-0.03516384000000, 0, 0.08832942000000, 0.23389032000000, -0.76027237000000, 0.58751830000000, 0, -0.11430184000000, 0, 0])
    imag_rec_lo, imag_rec_hi = imag_dec_lo[::-1], imag_dec_hi[::-1]

    real_fb = [real_dec_lo, real_dec_hi, real_rec_lo, real_rec_hi]
    imag_fb = [imag_dec_lo, imag_dec_hi, imag_rec_lo, imag_rec_hi]

    return Wavelet(name = 'real:', filter_bank = real_fb), Wavelet(name = 'imag:', filter_bank = imag_fb)<commit_msg>COMPAT: clearer code<commit_after>"""
Extension of PyWavelets to complex wavelets suitable for the Dual-Tree Complex Wavelet Transform.

Author : Laurent P. René de Cotret
"""
import numpy as n
from os.path import join, dirname
from pywt import Wavelet, wavelist
from warnings import warn

# lru_cache only exists as of Python 3.2
def _trivial_decorator(func, *args, **kwargs):
    return func
try:
    from functools import lru_cache
except ImportError:
    warn('functools.lru_cache could not be found. Performance will be affected.', ImportWarning)
    lru_cache = _trivial_decorator

__all__ = ['dualtree_wavelet', 'dualtree_first_stage']

DATADIR = join(dirname(__file__), 'data')
ALL_QSHIFT = ('qshift1', 'qshift2', 'qshift3', 'qshift4')
ALL_COMPLEX_WAV = ('kingsbury99',) + ALL_QSHIFT
ALL_FIRST_STAGE = ('kingsbury99_fs',) + tuple([wav for wav in wavelist() if wav != 'dmey'])

@lru_cache(maxsize = len(ALL_COMPLEX_WAV))
def dualtree_wavelet(name):
    """
    Returns a complex wavelet suitable for dual-tree cwt from a name.

    Parameters
    ----------
    name : str, {'qshift1', 'qshift2', 'qshift3', 'qshift4', 'kingsbury99'}
        Valid arguments can be found in dualtree.ALL_COMPLEX_WAV

    Returns
    -------
    real, imag : pywt.Wavelet objects.

    Raises
    ------
    ValueError
        If illegal wavelet name.
    """
    if name not in ALL_COMPLEX_WAV:
        raise ValueError('{} is not associated with any implemented complex wavelet. Possible choices are {}'.format(name, ALL_COMPLEX_WAV))

    # This function is simply for now
    if name == 'kingsbury99':
        return kingsbury99()

    return _qshift(name)

@lru_cache(maxsize = len(ALL_FIRST_STAGE))
def dualtree_first_stage(wavelet = 'kingsbury99_fs'):
    """
    Returns two wavelets to be used in the dual-tree complex wavelet transform, at the first stage.

    Parameters
    ----------
    wavelet : str or Wavelet
        Wavelet to be shifted for first-stage use. Valid arguments can be found in dualtree.ALL_FIRST_STAGE

    Return
    ------
    wav1, wav2 : Wavelet objects

    Raises
    ------
    ValueError
        If invalid first stage wavelet.
    """
    # Special case, preshifted
    if wavelet == 'kingsbury99_fs':
        return kingsbury99_fs()

    if not isinstance(wavelet, Wavelet):
        wavelet = Wavelet(wavelet)

    if wavelet.name not in ALL_FIRST_STAGE:
        raise ValueError('{} is an invalid first stage wavelet.'.format(wavelet.name))

    # extend filter bank with zeros
    filter_bank = [n.array(f, copy = True) for f in wavelet.filter_bank]
    for filt in filter_bank:
        extended = n.zeros( shape = (filt.shape[0] + 2,), dtype = n.float)
        extended[1:-1] = filt
        filt = extended

    # Shift deconstruction filters to one side, and reconstruction
    # to the other side
    shifted_fb = [n.array(f, copy = True) for f in wavelet.filter_bank]
    for filt in shifted_fb[::2]:    #Deconstruction filters
        filt = n.roll(filt, 1)
    for filt in shifted_fb[2::]:    # Reconstruction filters
        filt = n.roll(filt, -1)

    return Wavelet(name = wavelet.name, filter_bank = filter_bank), Wavelet(name = wavelet.name, filter_bank = shifted_fb)

@lru_cache(maxsize = len(ALL_QSHIFT))
def _qshift(name):
    """
    Returns a complex qshift wavelet by name.

    Parameters
    ----------
    name : str
        Wavelet to use. Valid arguments can be found in dualtree.ALL_QSHIFT

    Returns
    -------
    wav1, wav2 : pywt.Wavelet objects
        real and imaginary wavelet

    Raises
    ------
    ValueError
        If illegal wavelet family name.
    """
    filters = ('h0a', 'h0b', 'g0a', 'g0b', 'h1a', 'h1b', 'g1a', 'g1b')

    filename = join(DATADIR, name + '.npz')
    with n.load(filename) as mat:
        try:
            (dec_real_low, dec_imag_low, rec_real_low, rec_imag_low,
             dec_real_high, dec_imag_high, rec_real_high, rec_imag_high) = tuple([mat[k].flatten() for k in filters])
        except KeyError:
            raise ValueError('Wavelet does not define ({0}) coefficients'.format(', '.join(filters)))

    real_filter_bank = [dec_real_low, dec_real_high, rec_real_low, rec_real_high]
    imag_filter_bank = [dec_imag_low, dec_imag_high, rec_imag_low, rec_imag_high]

    return Wavelet(name = 'real:' + name, filter_bank = real_filter_bank), Wavelet(name = 'imag:' + name, filter_bank = imag_filter_bank)

#############################################################################################
#                           EXAMPLE COMPLEX WAVELETS FROM
#               http://eeweb.poly.edu/iselesni/WaveletSoftware/dt1D.html
#############################################################################################
@lru_cache(maxsize = 1)
def kingsbury99_fs():
    """
    Returns a first-stage complex wavelet as published in [1].

    References
    ----------
    [1] Kingsbury, N. 'Image Processing with Complex Wavelets'. Philocophical Transactions of the Royal Society A pp. 2543-2560, September 1999
    """
    real_dec_lo = n.array([0, -0.08838834764832, 0.08838834764832, 0.69587998903400,0.69587998903400, 0.08838834764832, -0.08838834764832, 0.01122679215254, 0.01122679215254, 0])
    real_dec_hi = n.array([0, -0.01122679215254, 0.01122679215254, 0.08838834764832, 0.08838834764832, -0.69587998903400, 0.69587998903400, -0.08838834764832, -0.08838834764832, 0])
    real_rec_lo, real_rec_hi = real_dec_lo[::-1], real_dec_hi[::-1]

    imag_dec_lo = n.array([0.01122679215254, 0.01122679215254, -0.08838834764832, 0.08838834764832, 0.69587998903400, 0.69587998903400, 0.08838834764832, -0.08838834764832, 0, 0])
    imag_dec_hi = n.array([0, 0, -0.08838834764832, -0.08838834764832, 0.69587998903400, -0.69587998903400, 0.08838834764832, 0.08838834764832, 0.01122679215254, -0.01122679215254])
    imag_rec_lo, imag_rec_hi = imag_dec_lo[::-1], imag_dec_hi[::-1]

    real_fb = [real_dec_lo, real_dec_hi, real_rec_lo, real_rec_hi]
    imag_fb = [imag_dec_lo, imag_dec_hi, imag_rec_lo, imag_rec_hi]

    return Wavelet(name = 'real:', filter_bank = real_fb), Wavelet(name = 'imag:', filter_bank = imag_fb)

@lru_cache(maxsize = 1)
def kingsbury99():
    """
    Returns a late-stage complex wavelet as published in [1].

    References
    ----------
    [1] Kingsbury, N. 'Image Processing with Complex Wavelets'. Philocophical Transactions of the Royal Society A pp. 2543-2560, September 1999
    """
    real_dec_lo = n.array([ 0.03516384000000, 0, -0.08832942000000, 0.23389032000000, 0.76027237000000, 0.58751830000000, 0, -0.11430184000000, 0, 0])
    real_dec_hi = n.array([0, 0, -0.11430184000000, 0, 0.58751830000000, -0.76027237000000, 0.23389032000000, 0.08832942000000, 0, -0.03516384000000])
    real_rec_lo, real_rec_hi = real_dec_lo[::-1], real_dec_hi[::-1]

    imag_dec_lo = n.array([ 0, 0, -0.11430184000000, 0, 0.58751830000000, 0.76027237000000, 0.23389032000000, -0.08832942000000, 0, 0.03516384000000])
    imag_dec_hi = n.array([-0.03516384000000, 0, 0.08832942000000, 0.23389032000000, -0.76027237000000, 0.58751830000000, 0, -0.11430184000000, 0, 0])
    imag_rec_lo, imag_rec_hi = imag_dec_lo[::-1], imag_dec_hi[::-1]

    real_fb = [real_dec_lo, real_dec_hi, real_rec_lo, real_rec_hi]
    imag_fb = [imag_dec_lo, imag_dec_hi, imag_rec_lo, imag_rec_hi]

    return Wavelet(name = 'real:', filter_bank = real_fb), Wavelet(name = 'imag:', filter_bank = imag_fb)<|endoftext|>


<commit_before>import os

from . import constants
from .source import Repo
from .compiler.compose.common import container_code_path
from .systems.docker.common import spec_for_service
from .systems.rsync import sync_local_path_to_vm
from .path import parent_dir

def _write_commands_to_file(list_of_commands, file_location):
    file_location_parent = parent_dir(file_location)
    if not os.path.exists(file_location_parent):
        os.makedirs(file_location_parent)
    with open(file_location, 'w+') as f:
        for command in list_of_commands:
            f.write('{} \n'.format(command))

def _get_once_commands(app_spec):
    once_commands = app_spec['commands']['once']
    commands_with_function = []
    if once_commands:
        commands_with_function.append('dusty_once_fn () {')
        commands_with_function += once_commands
        commands_with_function.append('}')
    commands_with_function.append("if [ ! -f {} ]".format(constants.FIRST_RUN_FILE_PATH))
    commands_with_function.append("then mkdir -p {}; touch {}".format(constants.RUN_DIR, constants.FIRST_RUN_FILE_PATH))
    if once_commands:
        commands_with_function.append("dusty_once_fn | tee {}".format(constants.ONCE_LOG_PATH))
    commands_with_function.append("fi")
    return commands_with_function

def _get_always_commands(app_spec):
    commands_with_function = []
    always_commands = app_spec['commands']['always']
    if always_commands:
        commands_with_function.append('dusty_always_fn () {')
        commands_with_function += always_commands
        commands_with_function.append('}')
        commands_with_function.append('dusty_always_fn | tee {}'.format(constants.ALWAYS_LOG_PATH))
    return commands_with_function

def _compile_docker_commands(app_name, assembled_specs):
    """ This is used to compile the command that will be run when the docker container starts
    up. This command has to install any libs that the app uses, run the `always` command, and
    run the `once` command if the container is being launched for the first time """
    app_spec = assembled_specs['apps'][app_name]
    commands = _lib_install_commands_for_app(app_name, assembled_specs)
    commands.append("cd {}".format(container_code_path(app_spec)))
    commands.append("export PATH=$PATH:{}".format(container_code_path(app_spec)))
    commands += _get_once_commands(app_spec)
    commands += _get_always_commands(app_spec)
    return commands

def _get_test_image_setup_commands(app_or_lib_name, expanded_specs, testing_spec):
    commands = lib_install_commands_for_app_or_lib(app_or_lib_name, expanded_specs)
    commands += ['cd {}'.format(container_code_path(spec_for_service(app_or_lib_name, expanded_specs)))]
    commands += testing_spec['once']
    return commands

def _lib_install_commands_for_libs(assembled_specs, libs):
    commands = []
    for lib in libs:
        lib_spec = assembled_specs['libs'][lib]
        install_commands = _lib_install_commands(lib_spec)
        if install_commands:
            commands += install_commands
    return commands

def _lib_install_commands_for_app(app_name, assembled_specs):
    """ This returns a list of all the commands that will install libraries for a
    given app """
    libs = assembled_specs['apps'][app_name]['depends']['libs']
    return _lib_install_commands_for_libs(assembled_specs, libs)

def _lib_install_commands_for_lib(app_name, assembled_specs):
    """ This returns a list of all the commands that will install libraries for a
    given lib """
    libs = assembled_specs['libs'][app_name]['depends']['libs']
    return _lib_install_commands_for_libs(assembled_specs, libs)

def lib_install_commands_for_app_or_lib(app_or_lib_name, assembled_specs):
    if app_or_lib_name in assembled_specs['apps']:
        return _lib_install_commands_for_app(app_or_lib_name, assembled_specs)
    return _lib_install_commands_for_lib(app_or_lib_name, assembled_specs)

def _lib_install_commands(lib_spec):
    """ This returns a single commmand that will install a library in a docker container """
    if not lib_spec['install']:
        return []
    return ["cd {}".format(lib_spec['mount'])] + lib_spec['install']

def dusty_command_file_name(app_or_lib_name, script_name=None, test_name=None):
    command_file_name = 'dusty_command_file_{}'.format(app_or_lib_name)
    if script_name:
        command_file_name = '{}_script_{}'.format(command_file_name, script_name)
    elif test_name:
        command_file_name = '{}_test_{}'.format(command_file_name, test_name)
    return "{}.sh".format(command_file_name)

def _write_up_command(app_name, assembled_specs):
    commands = _compile_docker_commands(app_name, assembled_specs)
    command_file_name = dusty_command_file_name(app_name)
    local_path = '{}/{}/{}'.format(constants.COMMAND_FILES_DIR, app_name, command_file_name)
    _write_commands_to_file(commands, local_path)

def _write_up_script_command(app_name, app_spec, script_spec):
    commands = ["cd {}".format(container_code_path(app_spec))] + script_spec['command']
    commands[-1] = '{} $@'.format(commands[-1])
    command_file_name = dusty_command_file_name(app_name, script_name=script_spec['name'])
    local_path = '{}/{}/{}'.format(constants.COMMAND_FILES_DIR, app_name, command_file_name)
    _write_commands_to_file(commands, local_path)

def _write_test_command(app_or_lib_spec, expanded_specs):
    commands = _get_test_image_setup_commands(app_or_lib_spec.name, expanded_specs, app_or_lib_spec['test'])
    command_file_name = dusty_command_file_name(app_or_lib_spec.name)
    local_path = '{}/{}/test/{}'.format(constants.COMMAND_FILES_DIR, app_or_lib_spec.name, command_file_name)
    _write_commands_to_file(commands, local_path)

def _write_test_suite_command(app_or_lib_spec, suite_spec):
    commands = ["cd {}".format(container_code_path(app_or_lib_spec))] + suite_spec['command']
    commands[-1] = '{} $@'.format(commands[-1])
    command_file_name = dusty_command_file_name(app_or_lib_spec.name, test_name=suite_spec['name'])
    local_path = '{}/{}/test/{}'.format(constants.COMMAND_FILES_DIR, app_or_lib_spec.name, command_file_name)
    _write_commands_to_file(commands, local_path)

def make_up_command_files(assembled_specs):
    for app_name in assembled_specs['apps'].keys():
        spec = assembled_specs['apps'][app_name]
        _write_up_command(app_name, assembled_specs)
        script_specs = spec['scripts']
        for script_spec in script_specs:
            _write_up_script_command(app_name, spec, script_spec)
    sync_local_path_to_vm(constants.COMMAND_FILES_DIR, constants.VM_COMMAND_FILES_DIR)

def make_test_command_files(expanded_specs):
    for app_or_lib_spec in expanded_specs.get_apps_and_libs():
        test_spec = app_or_lib_spec['test']
        if test_spec and test_spec['once']:
            suite_specs = test_spec['suites']
            _write_test_command(app_or_lib_spec, expanded_specs)
            for suite_spec in suite_specs:
                _write_test_suite_command(app_or_lib_spec, suite_spec)
    sync_local_path_to_vm(constants.COMMAND_FILES_DIR, constants.VM_COMMAND_FILES_DIR)
<commit_msg>break up line w semicolon<commit_after>import os

from . import constants
from .source import Repo
from .compiler.compose.common import container_code_path
from .systems.docker.common import spec_for_service
from .systems.rsync import sync_local_path_to_vm
from .path import parent_dir

def _write_commands_to_file(list_of_commands, file_location):
    file_location_parent = parent_dir(file_location)
    if not os.path.exists(file_location_parent):
        os.makedirs(file_location_parent)
    with open(file_location, 'w+') as f:
        for command in list_of_commands:
            f.write('{} \n'.format(command))

def _get_once_commands(app_spec):
    once_commands = app_spec['commands']['once']
    commands_with_function = []
    if once_commands:
        commands_with_function.append('dusty_once_fn () {')
        commands_with_function += once_commands
        commands_with_function.append('}')
    commands_with_function.append("if [ ! -f {} ]".format(constants.FIRST_RUN_FILE_PATH))
    commands_with_function.append("then mkdir -p {}".format(constants.RUN_DIR))
    commands_with_function.append("touch {}".format(constants.FIRST_RUN_FILE_PATH))
    if once_commands:
        commands_with_function.append("dusty_once_fn | tee {}".format(constants.ONCE_LOG_PATH))
    commands_with_function.append("fi")
    return commands_with_function

def _get_always_commands(app_spec):
    commands_with_function = []
    always_commands = app_spec['commands']['always']
    if always_commands:
        commands_with_function.append('dusty_always_fn () {')
        commands_with_function += always_commands
        commands_with_function.append('}')
        commands_with_function.append('dusty_always_fn | tee {}'.format(constants.ALWAYS_LOG_PATH))
    return commands_with_function

def _compile_docker_commands(app_name, assembled_specs):
    """ This is used to compile the command that will be run when the docker container starts
    up. This command has to install any libs that the app uses, run the `always` command, and
    run the `once` command if the container is being launched for the first time """
    app_spec = assembled_specs['apps'][app_name]
    commands = _lib_install_commands_for_app(app_name, assembled_specs)
    commands.append("cd {}".format(container_code_path(app_spec)))
    commands.append("export PATH=$PATH:{}".format(container_code_path(app_spec)))
    commands += _get_once_commands(app_spec)
    commands += _get_always_commands(app_spec)
    return commands

def _get_test_image_setup_commands(app_or_lib_name, expanded_specs, testing_spec):
    commands = lib_install_commands_for_app_or_lib(app_or_lib_name, expanded_specs)
    commands += ['cd {}'.format(container_code_path(spec_for_service(app_or_lib_name, expanded_specs)))]
    commands += testing_spec['once']
    return commands

def _lib_install_commands_for_libs(assembled_specs, libs):
    commands = []
    for lib in libs:
        lib_spec = assembled_specs['libs'][lib]
        install_commands = _lib_install_commands(lib_spec)
        if install_commands:
            commands += install_commands
    return commands

def _lib_install_commands_for_app(app_name, assembled_specs):
    """ This returns a list of all the commands that will install libraries for a
    given app """
    libs = assembled_specs['apps'][app_name]['depends']['libs']
    return _lib_install_commands_for_libs(assembled_specs, libs)

def _lib_install_commands_for_lib(app_name, assembled_specs):
    """ This returns a list of all the commands that will install libraries for a
    given lib """
    libs = assembled_specs['libs'][app_name]['depends']['libs']
    return _lib_install_commands_for_libs(assembled_specs, libs)

def lib_install_commands_for_app_or_lib(app_or_lib_name, assembled_specs):
    if app_or_lib_name in assembled_specs['apps']:
        return _lib_install_commands_for_app(app_or_lib_name, assembled_specs)
    return _lib_install_commands_for_lib(app_or_lib_name, assembled_specs)

def _lib_install_commands(lib_spec):
    """ This returns a single commmand that will install a library in a docker container """
    if not lib_spec['install']:
        return []
    return ["cd {}".format(lib_spec['mount'])] + lib_spec['install']

def dusty_command_file_name(app_or_lib_name, script_name=None, test_name=None):
    command_file_name = 'dusty_command_file_{}'.format(app_or_lib_name)
    if script_name:
        command_file_name = '{}_script_{}'.format(command_file_name, script_name)
    elif test_name:
        command_file_name = '{}_test_{}'.format(command_file_name, test_name)
    return "{}.sh".format(command_file_name)

def _write_up_command(app_name, assembled_specs):
    commands = _compile_docker_commands(app_name, assembled_specs)
    command_file_name = dusty_command_file_name(app_name)
    local_path = '{}/{}/{}'.format(constants.COMMAND_FILES_DIR, app_name, command_file_name)
    _write_commands_to_file(commands, local_path)

def _write_up_script_command(app_name, app_spec, script_spec):
    commands = ["cd {}".format(container_code_path(app_spec))] + script_spec['command']
    commands[-1] = '{} $@'.format(commands[-1])
    command_file_name = dusty_command_file_name(app_name, script_name=script_spec['name'])
    local_path = '{}/{}/{}'.format(constants.COMMAND_FILES_DIR, app_name, command_file_name)
    _write_commands_to_file(commands, local_path)

def _write_test_command(app_or_lib_spec, expanded_specs):
    commands = _get_test_image_setup_commands(app_or_lib_spec.name, expanded_specs, app_or_lib_spec['test'])
    command_file_name = dusty_command_file_name(app_or_lib_spec.name)
    local_path = '{}/{}/test/{}'.format(constants.COMMAND_FILES_DIR, app_or_lib_spec.name, command_file_name)
    _write_commands_to_file(commands, local_path)

def _write_test_suite_command(app_or_lib_spec, suite_spec):
    commands = ["cd {}".format(container_code_path(app_or_lib_spec))] + suite_spec['command']
    commands[-1] = '{} $@'.format(commands[-1])
    command_file_name = dusty_command_file_name(app_or_lib_spec.name, test_name=suite_spec['name'])
    local_path = '{}/{}/test/{}'.format(constants.COMMAND_FILES_DIR, app_or_lib_spec.name, command_file_name)
    _write_commands_to_file(commands, local_path)

def make_up_command_files(assembled_specs):
    for app_name in assembled_specs['apps'].keys():
        spec = assembled_specs['apps'][app_name]
        _write_up_command(app_name, assembled_specs)
        script_specs = spec['scripts']
        for script_spec in script_specs:
            _write_up_script_command(app_name, spec, script_spec)
    sync_local_path_to_vm(constants.COMMAND_FILES_DIR, constants.VM_COMMAND_FILES_DIR)

def make_test_command_files(expanded_specs):
    for app_or_lib_spec in expanded_specs.get_apps_and_libs():
        test_spec = app_or_lib_spec['test']
        if test_spec and test_spec['once']:
            suite_specs = test_spec['suites']
            _write_test_command(app_or_lib_spec, expanded_specs)
            for suite_spec in suite_specs:
                _write_test_suite_command(app_or_lib_spec, suite_spec)
    sync_local_path_to_vm(constants.COMMAND_FILES_DIR, constants.VM_COMMAND_FILES_DIR)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import os
import stat
import uuid

import dvc.logger as logger
from dvc.system import System
from dvc.utils import copyfile, move, remove
from dvc.exceptions import DvcException

def _unprotect_file(path):
    if System.is_symlink(path) or System.is_hardlink(path):
        logger.debug("Unprotecting '{}'".format(path))
        tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))

        # The operations order is important here - if some application would
        # access the file during the process of copyfile then it would get
        # only the part of file. So, at first, the file should be copied with
        # the temporary name, and then original file should be replaced by new.
        copyfile(path, tmp)
        remove(path)
        os.rename(tmp, path)

    else:
        logger.debug(
            "Skipping copying for '{}', since it is not "
            "a symlink or a hardlink.".format(path)
        )

    os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)

def _unprotect_dir(path):
    for root, dirs, files in os.walk(str(path)):
        for f in files:
            path = os.path.join(root, f)
            _unprotect_file(path)

def unprotect(path):
    if not os.path.exists(path):
        raise DvcException(
            "can't unprotect non-existing data '{}'".format(path)
        )

    if os.path.isdir(path):
        _unprotect_dir(path)
    else:
        _unprotect_file(path)
<commit_msg>repo: unprotect: remove unused move<commit_after>from __future__ import unicode_literals

import os
import stat
import uuid

import dvc.logger as logger
from dvc.system import System
from dvc.utils import copyfile, remove
from dvc.exceptions import DvcException

def _unprotect_file(path):
    if System.is_symlink(path) or System.is_hardlink(path):
        logger.debug("Unprotecting '{}'".format(path))
        tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))

        # The operations order is important here - if some application would
        # access the file during the process of copyfile then it would get
        # only the part of file. So, at first, the file should be copied with
        # the temporary name, and then original file should be replaced by new.
        copyfile(path, tmp)
        remove(path)
        os.rename(tmp, path)

    else:
        logger.debug(
            "Skipping copying for '{}', since it is not "
            "a symlink or a hardlink.".format(path)
        )

    os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)

def _unprotect_dir(path):
    for root, dirs, files in os.walk(str(path)):
        for f in files:
            path = os.path.join(root, f)
            _unprotect_file(path)

def unprotect(path):
    if not os.path.exists(path):
        raise DvcException(
            "can't unprotect non-existing data '{}'".format(path)
        )

    if os.path.isdir(path):
        _unprotect_dir(path)
    else:
        _unprotect_file(path)
<|endoftext|>


<commit_before>import functools
import json
import os

def absdir(directory, base_dir):
    if not directory.startswith('/'):
        directory = os.path.join(base_dir, directory)
    return os.path.normpath(directory)

def dirs(list_dir, base_dir):
    result = []
    for i in list_dir:
        result.append(absdir(i, base_dir))
    return result

def convert_to_djangodb(d, name, base_dir='/tmp'):
    if d.get('database'):
        db = {
            k.upper(): v
            for k, v in d.items()
            if v}
        if db.pop('GIS', None):
            db['ENGINE'] = 'django.contrib.gis.db.backends.postgis'
        else:
            db['ENGINE'] = 'django.db.backends.postgresql_psycopg2'
        db['NAME'] = db.pop('DATABASE')
        # Use same db name for test. Use custom config for tests to separate test and dev dbs.
        db['TEST'] = {'NAME': db['NAME']}
    else:
        return {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': os.path.join(base_dir, name + '.sqlite3'),
        }
    return db

def db_to_settings(db_dict, base_dir):
    return {
        n: convert_to_djangodb(v, n, base_dir=base_dir)
        for n, v in db_dict.items()
    }

def convert_to_django_redis(config):
    return {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': 'redis://{host}:{port}/{db}'.format(**config),
        'OPTIONS': {
            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
        }
    }

def redis_to_settings(redis_dict):
    return {name: convert_to_django_redis(value) for name, value in redis_dict.items()}

async def cleanup_ctx_redis(app, cfg_key='default', app_key='redis'):
    import aioredis
    cfg = app.context.config.redis[cfg_key].get('connection', {})
    address = cfg.pop('host', 'localhost'), cfg.pop('port', 6379)
    pool = await aioredis.create_redis_pool(
        address,
        db=cfg.get('db', 0),
        minsize=cfg.get('minsize', 1),
        maxsize=cfg.get('maxsize', 10),
        loop=app.loop)
    app[app_key] = pool
    yield
    pool.close()
    await pool.wait_closed()

cleanup_ctx_redis_sessions = functools.partial(
    cleanup_ctx_redis, app_key='sessions', cfg_key='sessions')

async def cleanup_ctx_aiopg(app, cfg_key='default', app_key='db'):
    import aiopg.sa
    from dvhb_hybrid.amodels import AppModels
    dbparams = app.context.config.databases.get(cfg_key)
    app.models = app.m = AppModels(app)
    async with aiopg.sa.create_engine(dbparams.uri) as pool:
        app[app_key] = pool
        yield

async def cleanup_ctx_databases(app, cfg_key='default', app_key='db'):
    import asyncpgsa
    from dvhb_hybrid.amodels import AppModels

    app.models = app.m = AppModels(app)

    async def init(connection):
        for t in ['json', 'jsonb']:
            await connection.set_type_codec(
                t,
                encoder=lambda x: x,
                decoder=json.loads,
                schema='pg_catalog',
            )
    dbparams = app.context.config.databases.get(cfg_key)
    if 'uri' in dbparams:
        dbargs, dbkwargs = (dbparams.uri,), {}
    else:
        dbargs, dbkwargs = (), dbparams

    async with asyncpgsa.create_pool(*dbargs, init=init, **dbkwargs) as pool:
        app[app_key] = pool
        yield
<commit_msg>fix cleanup_ctx_redis<commit_after>import functools
import json
import os

def absdir(directory, base_dir):
    if not directory.startswith('/'):
        directory = os.path.join(base_dir, directory)
    return os.path.normpath(directory)

def dirs(list_dir, base_dir):
    result = []
    for i in list_dir:
        result.append(absdir(i, base_dir))
    return result

def convert_to_djangodb(d, name, base_dir='/tmp'):
    if d.get('database'):
        db = {
            k.upper(): v
            for k, v in d.items()
            if v}
        if db.pop('GIS', None):
            db['ENGINE'] = 'django.contrib.gis.db.backends.postgis'
        else:
            db['ENGINE'] = 'django.db.backends.postgresql_psycopg2'
        db['NAME'] = db.pop('DATABASE')
        # Use same db name for test. Use custom config for tests to separate test and dev dbs.
        db['TEST'] = {'NAME': db['NAME']}
    else:
        return {
            'ENGINE': 'django.db.backends.sqlite3',
            'NAME': os.path.join(base_dir, name + '.sqlite3'),
        }
    return db

def db_to_settings(db_dict, base_dir):
    return {
        n: convert_to_djangodb(v, n, base_dir=base_dir)
        for n, v in db_dict.items()
    }

def convert_to_django_redis(config):
    return {
        'BACKEND': 'django_redis.cache.RedisCache',
        'LOCATION': 'redis://{host}:{port}/{db}'.format(**config),
        'OPTIONS': {
            'CLIENT_CLASS': 'django_redis.client.DefaultClient',
        }
    }

def redis_to_settings(redis_dict):
    return {name: convert_to_django_redis(value) for name, value in redis_dict.items()}

async def cleanup_ctx_redis(app, cfg_key='default', app_key='redis'):
    import aioredis
    cfg = app.context.config.redis[cfg_key].get('connection', {})
    address = cfg.get('host', 'localhost'), cfg.get('port', 6379)
    pool = await aioredis.create_redis_pool(
        address,
        db=cfg.get('db', 0),
        minsize=cfg.get('minsize', 1),
        maxsize=cfg.get('maxsize', 10),
        loop=app.loop)
    app[app_key] = pool
    yield
    pool.close()
    await pool.wait_closed()

cleanup_ctx_redis_sessions = functools.partial(
    cleanup_ctx_redis, app_key='sessions', cfg_key='sessions')

async def cleanup_ctx_aiopg(app, cfg_key='default', app_key='db'):
    import aiopg.sa
    from dvhb_hybrid.amodels import AppModels
    dbparams = app.context.config.databases.get(cfg_key)
    app.models = app.m = AppModels(app)
    async with aiopg.sa.create_engine(dbparams.uri) as pool:
        app[app_key] = pool
        yield

async def cleanup_ctx_databases(app, cfg_key='default', app_key='db'):
    import asyncpgsa
    from dvhb_hybrid.amodels import AppModels

    app.models = app.m = AppModels(app)

    async def init(connection):
        for t in ['json', 'jsonb']:
            await connection.set_type_codec(
                t,
                encoder=lambda x: x,
                decoder=json.loads,
                schema='pg_catalog',
            )
    dbparams = app.context.config.databases.get(cfg_key)
    if 'uri' in dbparams:
        dbargs, dbkwargs = (dbparams.uri,), {}
    else:
        dbargs, dbkwargs = (), dbparams

    async with asyncpgsa.create_pool(*dbargs, init=init, **dbkwargs) as pool:
        app[app_key] = pool
        yield
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import abc
import logging
import os
from dataclasses import dataclass
from typing import List, Tuple, Union
import json

import cv2
import numpy as np
import tiledb

logger = logging.getLogger(__name__)

class Dataset(abc.ABC):
    @property
    @abc.abstractmethod
    def shape(self) -> Tuple[int, int]:
        ...

    @property
    @abc.abstractmethod
    def tile_size(self) -> Tuple[int, int]:
        ...

    @property
    @abc.abstractmethod
    def dzi_sampling_level(self) -> int:
        ...

    @property
    @abc.abstractmethod
    def slide_path(self) -> str:
        ...

    @property
    @abc.abstractmethod
    def slide_resolution(self) -> Tuple[int, int]:
        ...

    @property
    @abc.abstractmethod
    def array(self) -> np.ndarray:
        ...

class TileDBDataset(Dataset):
    def __init__(self, array: tiledb.Array):
        self._array = tiledb.open(array.uri, mode="r")

    @property
    def shape(self):
        return self._array.shape

    @property
    def tile_size(self) -> Tuple[int, int]:
        return self._array.meta["tumor.tile_size"]

    @property
    def dzi_sampling_level(self) -> int:
        return self._array.meta["tumor.dzi_sampling_level"]

    @property
    def slide_path(self) -> str:
        return self._array.meta["slide_path"]

    @property
    def slide_resolution(self) -> Tuple[int, int]:
        return (self._array.meta["original_width"], self._array.meta["original_height"])

    @property
    def array(self) -> np.ndarray:
        return np.array(self._array)

def get_dataset(path):
    ext = os.path.splitext(path)[1]
    if ext == ".tiledb":
        return TileDBDataset(tiledb.open(path))
    else:
        raise UnsupportedDataset(path)

class UnsupportedDataset(Exception):
    pass

Coord = Union[int, float]
Point = Tuple[Coord, Coord]

@dataclass(frozen=True)
class Shape:
    points: Tuple[Point]

    def rescale(self, factor: float) -> "Shape":
        return Shape(tuple(map(tuple, np.array(self.points) * factor)))

    def __eq__(self, other):
        return set(self.points) == set(other.points)

    def as_points(self) -> List:
        points = [{"point": {"x": p[0], "y": p[1]}} for p in self.points]
        return points

def shapes_to_json(shapes: List[Shape]) -> str:
    return json.dumps({"shapes": shapes}, default=lambda s: s.as_points())

class ShapeConverter(abc.ABC):
    @abc.abstractmethod
    def convert(self, dataset: Dataset, threshold: float) -> List[Shape]:
        ...

class OpenCVShapeConverter(ShapeConverter):
    def convert(self, dataset: Dataset, threshold: float) -> List[Shape]:
        mask = dataset.array
        mask[mask <= threshold] = 0
        mask[mask > threshold] = 255

        contours, _ = cv2.findContours(
            mask, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_SIMPLE
        )
        logger.debug("contours %s", contours)
        factor = dataset.slide_resolution[0] / dataset.shape[0]
        points = [
            tuple(map(tuple, np.squeeze(c))) for c in contours
        ]
        points = [p + (p[0],) for p in points]
        shapes = [
            Shape(point).rescale(factor) for point in points
        ]
        return shapes

def get_shape_converter(cls: str):
    if cls == "opencv":
        return OpenCVShapeConverter()
    else:
        raise KeyError(f"shape converter {cls} does not exist")
<commit_msg>fix contours filtering the ones with less than 3 points<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import abc
import logging
import os
from dataclasses import dataclass
from typing import List, Tuple, Union
import json

import cv2
import numpy as np
import tiledb

logger = logging.getLogger(__name__)

class Dataset(abc.ABC):
    @property
    @abc.abstractmethod
    def shape(self) -> Tuple[int, int]:
        ...

    @property
    @abc.abstractmethod
    def tile_size(self) -> Tuple[int, int]:
        ...

    @property
    @abc.abstractmethod
    def dzi_sampling_level(self) -> int:
        ...

    @property
    @abc.abstractmethod
    def slide_path(self) -> str:
        ...

    @property
    @abc.abstractmethod
    def slide_resolution(self) -> Tuple[int, int]:
        ...

    @property
    @abc.abstractmethod
    def array(self) -> np.ndarray:
        ...

class TileDBDataset(Dataset):
    def __init__(self, array: tiledb.Array):
        self._array = tiledb.open(array.uri, mode="r")

    @property
    def shape(self):
        return self._array.shape

    @property
    def tile_size(self) -> Tuple[int, int]:
        return self._array.meta["tumor.tile_size"]

    @property
    def dzi_sampling_level(self) -> int:
        return self._array.meta["tumor.dzi_sampling_level"]

    @property
    def slide_path(self) -> str:
        return self._array.meta["slide_path"]

    @property
    def slide_resolution(self) -> Tuple[int, int]:
        return (self._array.meta["original_width"], self._array.meta["original_height"])

    @property
    def array(self) -> np.ndarray:
        return np.array(self._array)

def get_dataset(path):
    ext = os.path.splitext(path)[1]
    if ext == ".tiledb":
        return TileDBDataset(tiledb.open(path))
    else:
        raise UnsupportedDataset(path)

class UnsupportedDataset(Exception):
    pass

Coord = Union[int, float]
Point = Tuple[Coord, Coord]

@dataclass(frozen=True)
class Shape:
    points: Tuple[Point]

    def rescale(self, factor: float) -> "Shape":
        return Shape(tuple(map(tuple, np.array(self.points) * factor)))

    def __eq__(self, other):
        return set(self.points) == set(other.points)

    def as_points(self) -> List:
        points = [{"point": {"x": p[0], "y": p[1]}} for p in self.points]
        return points

def shapes_to_json(shapes: List[Shape]) -> str:
    return json.dumps({"shapes": shapes}, default=lambda s: s.as_points())

class ShapeConverter(abc.ABC):
    @abc.abstractmethod
    def convert(self, dataset: Dataset, threshold: float) -> List[Shape]:
        ...

class OpenCVShapeConverter(ShapeConverter):
    def convert(self, dataset: Dataset, threshold: float) -> List[Shape]:
        mask = dataset.array
        mask[mask <= threshold] = 0
        mask[mask > threshold] = 255

        contours, _ = cv2.findContours(
            mask, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_SIMPLE
        )
        factor = dataset.slide_resolution[0] / dataset.shape[0]
        points = [tuple(map(tuple, np.squeeze(c))) for c in contours if c.size >= 3]
        points = [p + (p[0],) for p in points]
        shapes = [Shape(point).rescale(factor) for point in points]
        return shapes

def get_shape_converter(cls: str):
    if cls == "opencv":
        return OpenCVShapeConverter()
    else:
        raise KeyError(f"shape converter {cls} does not exist")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import datetime, pprint
from django.http import QueryDict
from django.test import TestCase
from easyscan_app.models import LasDataMaker, ScanRequest, StatsBuilder
from easyscan_app.lib.magic_bus import Prepper
from easyscan_app.lib.spacer import Spacer

# maker = LasDataMaker()
prepper = Prepper()
statsbuilder = StatsBuilder()

class LasDataMakerTest( TestCase ):
    """ Tests models.LasDataMaker() """

    def setUp(self):
        self.maker = LasDataMaker()
        self.maxDiff = None

    def test__utf8list_to_utf8csv__str( self ):
        """ Tests good utf8 strings (required by csv module). """
        utf8_list = [ b'foo', b'bar', b'“iñtërnâtiônàlĭzætiøn”' ]
        result = self.maker.utf8list_to_utf8csv( utf8_list )
        self.assertEqual(
            b'"foo","bar","\xe2\x80\x9ci\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xc3\xa0l\xc4\xadz\xc3\xa6ti\xc3\xb8n\xe2\x80\x9d"\r\n',
            result )
        self.assertEqual(
            str,
            type(result) )

    def test__utf8list_to_utf8csv__unicode( self ):
        """ Tests bad unicode strings. """
        unicode_list = [ u'foo', u'bar', u'“iñtërnâtiônàlĭzætiøn”' ]
        result = u'init'
        try:
            self.maker.utf8list_to_utf8csv( unicode_list )
        except Exception as e:
            result = unicode(e)
        self.assertEqual(
            u"entry `u'foo'` not of type str",
            result )

    def test__make_date_string( self ):
        """ Tests conversion of datetime object to string required by LAS. """
        dt = datetime.datetime( 2014, 12, 8, 12, 40, 59 )
        self.assertEqual(
            u'Mon Dec 08 2014',
            self.maker.make_date_string( dt )
            )

    def test__strip_stuff( self ):
        """ Tests removal of double-quotes and new-lines. """
        self.assertEqual(
            u"The title was 'Zen', I think.",
            self.maker.strip_stuff(u'The title was "Zen", I think.') )
        self.assertEqual(
            u'first line - second line',
            self.maker.strip_stuff(u'first line\nsecond line') )
        self.assertEqual(
            u'first line - second line',
            self.maker.strip_stuff(u'first line\rsecond line') )
        self.assertEqual(
            u"The title was 'Zen', I think.",
            self.maker.strip_stuff(u'The title was `Zen`, I think.') )

    def test__add_email( self ):
        """ Checks for space before and after actual email line. """
        email = 'a@a.edu'
        expected_lst = [
            'PATRON_EMAIL...                                   ',  # 50 characters
            '                                                  ',
            'a@a.edu | A@A.EDU                                 ',
            '                                                  '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.maker.add_email( email )
            )

    def test__add_article_chapter_title( self ):
        """ Checks for space before and after article-chapter-title line. """
        initial_data = 'foo bar                                           '  # 50 characters
        article_chapter_title = 'Ultrastructural analysis of the effect of ethane dimethanesulphonate on the testis of the rat, guinea pig, hamster and mouse.'
        expected_lst = [
            'foo bar                                           ',  # 50 characters
            'ARTICLE-CHAPTER-TITLE...                          ',
            '                                                  ',
            'Ultrastructural analysis of the effect of ethane  ',
            'dimethanesulphonate on the testis of the rat,     ',
            'guinea pig, hamster and mouse.                    ',
            '                                                  '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.maker.add_article_chapter_title( initial_data, article_chapter_title )
            )

    def test__make_notes_field( self ):
        """ Checks for proper spacing. """
        patron_email = 'a@a.edu'
        item_chap_vol_title = 'test-article-title'
        item_page_range_other = 'test-range'
        item_other = 'test-other'
        expected_lst = [
            'PATRON_EMAIL...                                   ',  # 50 characters
            '                                                  ',
            'a@a.edu | A@A.EDU                                 ',
            '                                                  ',
            'ARTICLE-CHAPTER-TITLE...                          ',
            '                                                  ',
            'test-article-title                                ',
            '                                                  ',
            'PAGE-RANGE: test-range                            ',
            'PAGE-OTHER: test-other                            ',
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.maker.make_notes_field( patron_email, item_chap_vol_title, item_page_range_other, item_other )
            )

    # end class class LasDataMakerTest

class SpacerTest( TestCase ):
    """ Checks spacer.py Spacer() """

    def setUp(self):
        self.spcr = Spacer()

    ## convert_string_to_lines() ##

    def test__convert_string_to_lines__short_blank(self):
        self.spcr.notes_line_length = 4
        self.assertEqual(
            [ '' ],
            self.spcr.convert_string_to_lines( ' ' )
            )

    def test__convert_string_to_lines__short(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            [ 'abc' ],
            self.spcr.convert_string_to_lines( 'abc' )
            )

    def test__convert_string_to_lines__full(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            ['1234567890'],
            self.spcr.convert_string_to_lines( '1234567890' )
            )

    def test__convert_string_to_lines__single_bigger(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            [u'123456789012'],
            self.spcr.convert_string_to_lines( '123456789012' )
            )

    def test__convert_string_to_lines__single_bigger_plus_more_words(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            ['123456789012', 'aaa'],
            self.spcr.convert_string_to_lines( '123456789012 aaa' )
            )

    def test__convert_string_to_lines__bigger_even_break(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            ['1234567890', '123'],
            self.spcr.convert_string_to_lines( '1234567890 123' )
            )

    def test__convert_string_to_lines__bigger_uneven_break(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            ['12345678', '012'],
            self.spcr.convert_string_to_lines( '12345678 012' )
            )

    ## add_spacer() ##

    def test__add_spacer_small_string( self ):
        """ Tests filler in no-wrap situation. """
        self.spcr.notes_line_length = 10
        self.spcr.spacer_character = '|'
        self.assertEqual(
            'abc ||||| ',
            self.spcr.add_spacer( 'abc' )
            )

    def test__add_spacer_full_length_string( self ):
        """ Checks that full-string gets a full extra string added (ending in a space). """
        self.spcr.notes_line_length = 10
        self.spcr.spacer_character = '|'
        ten_characters = 'x' * 10
        expected_lst = [
            'xxxxxxxxxx',
            ' |||||||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( ten_characters )
            )

    def test__add_spacer_full_length_string_using_spaces( self ):
        """ Checks that full-string gets a full extra string added (ending in a space) when using expected spacer character of ' '. """
        self.spcr.notes_line_length = 10
        self.spcr.spacer_character = ' '
        ten_characters = 'x' * 10
        expected_lst = [
            'xxxxxxxxxx',
            '          '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( ten_characters )
            )

    def test__add_spacer_big_string( self ):
        """ Tests filler when wrapping. """
        self.spcr.notes_line_length = 10
        self.spcr.spacer_character = '|'
        fifteen_characters = 'x' * 15
        expected_lst = [
            'xxxxxxxxxx',
            'xxxxx ||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( fifteen_characters )
            )

    def test__add_spacer_big_string2( self ):
        """ Tests filler when wrapping will hit on a break. """
        self.spcr.notes_line_length = 50
        self.spcr.spacer_character = '|'
        long_text = '''A really long article title. A really long article title. A really long article title.'''
        expected_lst = [
            'A really long article title. A really long |||||| ',
            'article title. A really long article title. ||||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( long_text )
            )

    def test__add_spacer_big_string3( self ):
        """ Tests filler wrapping will hit on a break2. """
        self.spcr.notes_line_length = 50
        self.spcr.spacer_character = '|'
        long_text = '''First line will break after (46th letter) THIS aanndd then continue.'''
        expected_lst = [
            'First line will break after (46th letter) THIS || ',
            'aanndd then continue. ||||||||||||||||||||||||||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( long_text )
            )

    def test__add_spacer_big_string4( self ):
        """ Tests filler wrapping will hit on end-of-word.
            (Ends up breaking on the previous word.) """
        self.spcr.notes_line_length = 50
        self.spcr.spacer_character = '|'
        long_text = '''First line will break afterrrrr (50th letter) THIS and then continue.'''
        expected_lst = [
            'First line will break afterrrrr (50th letter) ||| ',
            'THIS and then continue. ||||||||||||||||||||||||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( long_text )
            )

    # end class SpacerTest()

class MagicBusPrepperTest( TestCase ):
    """ Tests magic_bus.py Prepper() """

    def test__make_filename_datestring( self ):
        """ Tests conversion of datetime object to string for filename. """
        dt = datetime.datetime( 2014, 12, 8, 15, 40, 59 )
        self.assertEqual(
            u'2014-12-08T15:40:59',
            prepper.make_filename_datestring( dt )
            )

    # end class MagicBusPrepperTest

class StatsBuilderTest( TestCase ):
    """ Tests models.py StatsBuilder() """

    def test__check_params( self ):
        """ Tests keys. """
        ## bad params
        qdict = QueryDict( u'', mutable=True ); qdict.update( {u'start': u'a', u'end': u'b'} )
        self.assertEqual( False, statsbuilder.check_params(qdict, u'server_name') )
        ## good params
        qdict = QueryDict( u'', mutable=True ); qdict.update( {u'start_date': 'a', u'end_date': 'b'} )
        self.assertEqual( True, statsbuilder.check_params(qdict, u'server_name') )

    def test__run_query( self ):
        """ Tests that scanrequest is found and returned. """
        sr = ScanRequest( item_title=u'foo' )
        sr.save()
        qdict = QueryDict( u'', mutable=True ); qdict.update( {u'start_date': datetime.date.today(), u'end_date': datetime.date.today()} )
        statsbuilder.check_params( qdict, u'server_name' )
        results = statsbuilder.run_query()
        self.assertEqual( 1, len(results) )

<commit_msg>adds failing test.<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals

import datetime, pprint
from django.http import QueryDict
from django.test import TestCase
from easyscan_app.models import LasDataMaker, ScanRequest, StatsBuilder
from easyscan_app.lib.magic_bus import Prepper
from easyscan_app.lib.spacer import Spacer

# maker = LasDataMaker()
prepper = Prepper()
statsbuilder = StatsBuilder()

class LasDataMakerTest( TestCase ):
    """ Tests models.LasDataMaker() """

    def setUp(self):
        self.maker = LasDataMaker()
        self.maxDiff = None

    def test__utf8list_to_utf8csv__str( self ):
        """ Tests good utf8 strings (required by csv module). """
        utf8_list = [ b'foo', b'bar', b'“iñtërnâtiônàlĭzætiøn”' ]
        result = self.maker.utf8list_to_utf8csv( utf8_list )
        self.assertEqual(
            b'"foo","bar","\xe2\x80\x9ci\xc3\xb1t\xc3\xabrn\xc3\xa2ti\xc3\xb4n\xc3\xa0l\xc4\xadz\xc3\xa6ti\xc3\xb8n\xe2\x80\x9d"\r\n',
            result )
        self.assertEqual(
            str,
            type(result) )

    def test__utf8list_to_utf8csv__unicode( self ):
        """ Tests bad unicode strings. """
        unicode_list = [ u'foo', u'bar', u'“iñtërnâtiônàlĭzætiøn”' ]
        result = u'init'
        try:
            self.maker.utf8list_to_utf8csv( unicode_list )
        except Exception as e:
            result = unicode(e)
        self.assertEqual(
            u"entry `u'foo'` not of type str",
            result )

    def test__make_date_string( self ):
        """ Tests conversion of datetime object to string required by LAS. """
        dt = datetime.datetime( 2014, 12, 8, 12, 40, 59 )
        self.assertEqual(
            u'Mon Dec 08 2014',
            self.maker.make_date_string( dt )
            )

    def test__strip_stuff( self ):
        """ Tests removal of double-quotes and new-lines. """
        self.assertEqual(
            u"The title was 'Zen', I think.",
            self.maker.strip_stuff(u'The title was "Zen", I think.') )
        self.assertEqual(
            u'first line - second line',
            self.maker.strip_stuff(u'first line\nsecond line') )
        self.assertEqual(
            u'first line - second line',
            self.maker.strip_stuff(u'first line\rsecond line') )
        self.assertEqual(
            u"The title was 'Zen', I think.",
            self.maker.strip_stuff(u'The title was `Zen`, I think.') )

    def test__add_email( self ):
        """ Checks for space before and after actual email line. """
        email = 'a@a.edu'
        expected_lst = [
            'PATRON_EMAIL...                                   ',  # 50 characters
            '                                                  ',
            'a@a.edu | A@A.EDU                                 ',
            '                                                  '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.maker.add_email( email )
            )

    def test__add_article_chapter_title( self ):
        """ Checks for space before and after article-chapter-title line. """
        initial_data = 'foo bar                                           '  # 50 characters
        article_chapter_title = 'Ultrastructural analysis of the effect of ethane dimethanesulphonate on the testis of the rat, guinea pig, hamster and mouse.'
        expected_lst = [
            'foo bar                                           ',  # 50 characters
            'ARTICLE-CHAPTER-TITLE...                          ',
            '                                                  ',
            'Ultrastructural analysis of the effect of ethane  ',
            'dimethanesulphonate on the testis of the rat,     ',
            'guinea pig, hamster and mouse.                    ',
            '                                                  '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.maker.add_article_chapter_title( initial_data, article_chapter_title )
            )

    def test__make_notes_field( self ):
        """ Checks for proper spacing. """
        patron_email = 'a@a.edu'
        item_chap_vol_title = 'test-article-title'
        item_page_range_other = 'test-range'
        item_other = 'test-other'
        expected_lst = [
            'PATRON_EMAIL...                                   ',  # 50 characters
            '                                                  ',
            'a@a.edu | A@A.EDU                                 ',
            '                                                  ',
            'ARTICLE-CHAPTER-TITLE...                          ',
            '                                                  ',
            'test-article-title                                ',
            '                                                  ',
            'PAGE-RANGE: test-range                            ',
            'PAGE-OTHER: test-other                            ',
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.maker.make_notes_field( patron_email, item_chap_vol_title, item_page_range_other, item_other )
            )

    # end class class LasDataMakerTest

class SpacerTest( TestCase ):
    """ Checks spacer.py Spacer() """

    def setUp(self):
        self.spcr = Spacer()

    ## convert_string_to_lines() ##

    def test__convert_string_to_lines__short_blank(self):
        self.spcr.notes_line_length = 4
        self.assertEqual(
            [ '' ],
            self.spcr.convert_string_to_lines( ' ' )
            )

    def test__convert_string_to_lines__short(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            [ 'abc' ],
            self.spcr.convert_string_to_lines( 'abc' )
            )

    def test__convert_string_to_lines__full(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            ['1234567890'],
            self.spcr.convert_string_to_lines( '1234567890' )
            )

    def test__convert_string_to_lines__single_bigger(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            [u'123456789012'],
            self.spcr.convert_string_to_lines( '123456789012' )
            )

    def test__convert_string_to_lines__single_bigger_plus_more_words(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            ['123456789012', 'aaa'],
            self.spcr.convert_string_to_lines( '123456789012 aaa' )
            )

    def test__convert_string_to_lines__bigger_even_break(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            ['1234567890', '123'],
            self.spcr.convert_string_to_lines( '1234567890 123' )
            )

    def test__convert_string_to_lines__bigger_uneven_break(self):
        self.spcr.notes_line_length = 10
        self.assertEqual(
            ['12345678', '012'],
            self.spcr.convert_string_to_lines( '12345678 012' )
            )

    def test__convert_string_to_lines__bigger_miscellaneous(self):
        self.spcr.notes_line_length = 50
        self.assertEqual( [
            'A surprisingly long TEST article-title, because',  # 47, would be 50
            'of the repetition of the surprisingly long',  # 42, would be 50
            'article title.'
            ],
            self.spcr.convert_string_to_lines( 'A surprisingly long TEST article-title, because of the repetition of the surprisingly long article title.' )
            )

    ## add_spacer() ##

    def test__add_spacer_small_string( self ):
        """ Tests filler in no-wrap situation. """
        self.spcr.notes_line_length = 10
        self.spcr.spacer_character = '|'
        self.assertEqual(
            'abc ||||| ',
            self.spcr.add_spacer( 'abc' )
            )

    def test__add_spacer_full_length_string( self ):
        """ Checks that full-string gets a full extra string added (ending in a space). """
        self.spcr.notes_line_length = 10
        self.spcr.spacer_character = '|'
        ten_characters = 'x' * 10
        expected_lst = [
            'xxxxxxxxxx',
            ' |||||||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( ten_characters )
            )

    def test__add_spacer_full_length_string_using_spaces( self ):
        """ Checks that full-string gets a full extra string added (ending in a space) when using expected spacer character of ' '. """
        self.spcr.notes_line_length = 10
        self.spcr.spacer_character = ' '
        ten_characters = 'x' * 10
        expected_lst = [
            'xxxxxxxxxx',
            '          '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( ten_characters )
            )

    def test__add_spacer_big_string( self ):
        """ Tests filler when wrapping. """
        self.spcr.notes_line_length = 10
        self.spcr.spacer_character = '|'
        fifteen_characters = 'x' * 15
        expected_lst = [
            'xxxxxxxxxx',
            'xxxxx ||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( fifteen_characters )
            )

    def test__add_spacer_big_string2( self ):
        """ Tests filler when wrapping will hit on a break. """
        self.spcr.notes_line_length = 50
        self.spcr.spacer_character = '|'
        long_text = '''A really long article title. A really long article title. A really long article title.'''
        expected_lst = [
            'A really long article title. A really long |||||| ',
            'article title. A really long article title. ||||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( long_text )
            )

    def test__add_spacer_big_string3( self ):
        """ Tests filler wrapping will hit on a break2. """
        self.spcr.notes_line_length = 50
        self.spcr.spacer_character = '|'
        long_text = '''First line will break after (46th letter) THIS aanndd then continue.'''
        expected_lst = [
            'First line will break after (46th letter) THIS || ',
            'aanndd then continue. ||||||||||||||||||||||||||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( long_text )
            )

    def test__add_spacer_big_string4( self ):
        """ Tests filler wrapping will hit on end-of-word.
            (Ends up breaking on the previous word.) """
        self.spcr.notes_line_length = 50
        self.spcr.spacer_character = '|'
        long_text = '''First line will break afterrrrr (50th letter) THIS and then continue.'''
        expected_lst = [
            'First line will break afterrrrr (50th letter) ||| ',
            'THIS and then continue. ||||||||||||||||||||||||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( long_text )
            )

    def test__add_spacer_big_string4( self ):
        """ This didn't wrap as expected... """
        self.spcr.notes_line_length = 50
        self.spcr.spacer_character = '|'
        long_text = 'A surprisingly long TEST article-title, because of the repetition of the surprisingly long article title.'
        expected_lst = [
            'First line will break afterrrrr (50th letter) ||| ',
            'THIS and then continue. ||||||||||||||||||||||||| '
            ]
        self.assertEqual(
            ''.join( expected_lst ),
            self.spcr.add_spacer( long_text )
            )

    # end class SpacerTest()

class MagicBusPrepperTest( TestCase ):
    """ Tests magic_bus.py Prepper() """

    def test__make_filename_datestring( self ):
        """ Tests conversion of datetime object to string for filename. """
        dt = datetime.datetime( 2014, 12, 8, 15, 40, 59 )
        self.assertEqual(
            u'2014-12-08T15:40:59',
            prepper.make_filename_datestring( dt )
            )

    # end class MagicBusPrepperTest

class StatsBuilderTest( TestCase ):
    """ Tests models.py StatsBuilder() """

    def test__check_params( self ):
        """ Tests keys. """
        ## bad params
        qdict = QueryDict( u'', mutable=True ); qdict.update( {u'start': u'a', u'end': u'b'} )
        self.assertEqual( False, statsbuilder.check_params(qdict, u'server_name') )
        ## good params
        qdict = QueryDict( u'', mutable=True ); qdict.update( {u'start_date': 'a', u'end_date': 'b'} )
        self.assertEqual( True, statsbuilder.check_params(qdict, u'server_name') )

    def test__run_query( self ):
        """ Tests that scanrequest is found and returned. """
        sr = ScanRequest( item_title=u'foo' )
        sr.save()
        qdict = QueryDict( u'', mutable=True ); qdict.update( {u'start_date': datetime.date.today(), u'end_date': datetime.date.today()} )
        statsbuilder.check_params( qdict, u'server_name' )
        results = statsbuilder.run_query()
        self.assertEqual( 1, len(results) )

<|endoftext|>


<commit_before>"""
This is the easystore that keeps configurations from multiple services in
a single store location. Uses a redis like interface, ideal to replace redis
with easystore.
"""
import os
import json

class DiskStore(object):
    """Initialize the DiskStore."""

    def __init__(self, storpath='/var/lib/easystore'):
        self.storpath = storpath
        self.__create_storpath()

    def getpath(self):
        return self.storpath

    def __create_storpath(self):
        try:
            if not os.path.exists(self.storpath):
                os.makedirs(self.storpath)
        except Exception:
            raise Exception("Issue with store path, check permissions")

    def setpath(self, storpath='/var/lib/easystore'):
        self.storpath = storpath
        self.__create_storpath()

    def hset(self, filename, keyname, kvdata):
        # TODO: count the number of arguments and raise error
        # TODO: Check if storpath ends with a / and re-eval below
        filepath = self.storpath + "/" + filename + "/" + keyname
        try:
            directory = os.path.dirname(filepath)
            if not os.path.exists(directory):
                os.makedirs(directory)
            with open(filepath) as json_file:
                json_data = json.load(json_file)
            json_data[keyname] = kvdata
        except IOError:
            json_data = dict()
            json_data[keyname] = kvdata

        with open(filepath, 'w') as json_file:
            json.dump(json_data, json_file)

    def set(self, filename, kvdata):
        # TODO: count the number of arguments and raise error
        # TODO: Check if storpath ends with a / and re-eval below
        filepath = self.storpath + "/" + filename
        try:
            json_data = kvdata
            with open(filepath, 'w') as json_file:
                json.dump(json_data, json_file)
        except IOError:
            return None

    def hget(self, filename, keyname):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename + "/" + keyname
        try:
            directory = os.path.dirname(filepath)
            if not os.path.exists(directory):
                os.makedirs(directory)
            with open(filepath) as json_file:
                json_data = json.load(json_file)
        except IOError:
            return None
        try:
            return json_data[keyname]
        except KeyError:
            return None

    def hdel(self, filename, keyname):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename + "/" + keyname
        try:
            with open(filepath) as json_file:
                json_data = json.load(json_file)
        except IOError:
            return None
        try:
            with open(filepath) as json_file:
                json_data = json.load(json_file)
            if os.path.isfile(filepath):
                os.remove(filepath)
            return json_data
        except KeyError:
            return None

    def get(self, filename):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename
        try:
            with open(filepath) as json_file:
                json_data = json.load(json_file)
        except IOError:
            return None
        try:
            return json_data
        except KeyError:
            return None

    def hmget(self, filename, storekeys):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename
        try:
            values = []
            for i in storekeys:
                directory = os.path.dirname(filepath + '/' + i)
                if not os.path.exists(directory):
                    os.makedirs(directory)
                with open(filepath + '/' + i) as json_file:
                    json_data = json.load(json_file)
                    values.append(json_data)
            return values
        except IOError:
            return None

    def hmset(self, filename, storekeys):
        # TODO: count the number of arguments and raise error
        # TODO: Check if storpath ends with a / and re-eval below
        filepath = self.storpath + "/" + filename
        try:
            for i in storekeys:
                directory = os.path.dirname(filepath + '/' + i)
                if not os.path.exists(directory):
                    os.makedirs(directory)
                with open(filepath + '/' + i, 'w') as json_file:
                    json.dump({i: storekeys[i]}, json_file)
        except IOError:
            return None

    def hgetall(self, filename):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename
        try:
            onlyfiles = [f for f in os.listdir(filepath) if os.path.isfile(os.path.join(filepath, f))]
            json_dict = {}
            for i in onlyfiles:
                with open(filepath + '/' + i) as json_file:
                    json_data = json.load(json_file)
                    json_dict.update(json_data)
        except IOError:
            return None
        if len(json_dict) > 0:
            return json_dict
        else:
            return None
<commit_msg>new error type encountered for hgetall when folder/main key is missing<commit_after>"""
This is the easystore that keeps configurations from multiple services in
a single store location. Uses a redis like interface, ideal to replace redis
with easystore.
"""
import os
import json

class DiskStore(object):
    """Initialize the DiskStore."""

    def __init__(self, storpath='/var/lib/easystore'):
        self.storpath = storpath
        self.__create_storpath()

    def getpath(self):
        return self.storpath

    def __create_storpath(self):
        try:
            if not os.path.exists(self.storpath):
                os.makedirs(self.storpath)
        except Exception:
            raise Exception("Issue with store path, check permissions")

    def setpath(self, storpath='/var/lib/easystore'):
        self.storpath = storpath
        self.__create_storpath()

    def hset(self, filename, keyname, kvdata):
        # TODO: count the number of arguments and raise error
        # TODO: Check if storpath ends with a / and re-eval below
        filepath = self.storpath + "/" + filename + "/" + keyname
        try:
            directory = os.path.dirname(filepath)
            if not os.path.exists(directory):
                os.makedirs(directory)
            with open(filepath) as json_file:
                json_data = json.load(json_file)
            json_data[keyname] = kvdata
        except IOError:
            json_data = dict()
            json_data[keyname] = kvdata

        with open(filepath, 'w') as json_file:
            json.dump(json_data, json_file)

    def set(self, filename, kvdata):
        # TODO: count the number of arguments and raise error
        # TODO: Check if storpath ends with a / and re-eval below
        filepath = self.storpath + "/" + filename
        try:
            json_data = kvdata
            with open(filepath, 'w') as json_file:
                json.dump(json_data, json_file)
        except IOError:
            return None

    def hget(self, filename, keyname):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename + "/" + keyname
        try:
            directory = os.path.dirname(filepath)
            if not os.path.exists(directory):
                os.makedirs(directory)
            with open(filepath) as json_file:
                json_data = json.load(json_file)
        except IOError:
            return None
        try:
            return json_data[keyname]
        except KeyError:
            return None

    def hdel(self, filename, keyname):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename + "/" + keyname
        try:
            with open(filepath) as json_file:
                json_data = json.load(json_file)
        except IOError:
            return None
        try:
            with open(filepath) as json_file:
                json_data = json.load(json_file)
            if os.path.isfile(filepath):
                os.remove(filepath)
            return json_data
        except KeyError:
            return None

    def get(self, filename):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename
        try:
            with open(filepath) as json_file:
                json_data = json.load(json_file)
        except IOError:
            return None
        try:
            return json_data
        except KeyError:
            return None

    def hmget(self, filename, storekeys):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename
        try:
            values = []
            for i in storekeys:
                directory = os.path.dirname(filepath + '/' + i)
                if not os.path.exists(directory):
                    os.makedirs(directory)
                with open(filepath + '/' + i) as json_file:
                    json_data = json.load(json_file)
                    values.append(json_data)
            return values
        except IOError:
            return None

    def hmset(self, filename, storekeys):
        # TODO: count the number of arguments and raise error
        # TODO: Check if storpath ends with a / and re-eval below
        filepath = self.storpath + "/" + filename
        try:
            for i in storekeys:
                directory = os.path.dirname(filepath + '/' + i)
                if not os.path.exists(directory):
                    os.makedirs(directory)
                with open(filepath + '/' + i, 'w') as json_file:
                    json.dump({i: storekeys[i]}, json_file)
        except IOError:
            return None

    def hgetall(self, filename):
        # TODO: count the number of arguments and raise error
        filepath = self.storpath + "/" + filename
        try:
            onlyfiles = [f for f in os.listdir(filepath) if os.path.isfile(os.path.join(filepath, f))]
            json_dict = {}
            for i in onlyfiles:
                with open(filepath + '/' + i) as json_file:
                    json_data = json.load(json_file)
                    json_dict.update(json_data)
        except IOError:
            return None
        except OSError:
            return None
        if len(json_dict) > 0:
            return json_dict
        else:
            return None
<|endoftext|>