<commit_before>#!/usr/bin/env pytest

from pathlib import Path

import logging

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

import htcondor

from ornithology import (
    action,
    config,
    Condor,
    ClusterState,
    JobStatus,
    format_script,
    write_file,
)

#
# FIXME: obviously, these could be parameterized.
#

@action
def test_script_contents():
    return format_script(
        """
        #!/usr/bin/env python3

        with open("new_output_file", "w") as f:
            f.write("")
        exit(0)
        """
    )

@action
def test_script(test_dir, test_script_contents):
    test_script = test_dir / "test_script.py"
    write_file(test_script, test_script_contents)
    return test_script

@action
def test_job_hash(test_dir, path_to_python, test_script):
    input_file = test_dir / "input_file"
    input_file.touch()
    return {
        "executable": path_to_python,
        "arguments": test_script,
        "transfer_executable": "true",
        "should_transfer_files": "true",
        "transfer_input_files": input_file,
        "universe": "vanilla",
        "log": "log",
        "output": "output",
        "error": "error",
    }

@action
def no_manifest_job_iwd(test_dir):
    iwd = test_dir / "completed_no_manifest_job"
    iwd.mkdir()
    return iwd

@action
def completed_no_manifest_job(no_manifest_job_iwd, default_condor, test_job_hash):
    cnmj = default_condor.submit({
        ** test_job_hash,
        "iwd": no_manifest_job_iwd,
        },
        count=1
    )
    assert cnmj.wait(
        condition=ClusterState.all_terminal,
        timeout=60,
        verbose=True,
        fail_condition=ClusterState.any_held,
    )
    return cnmj

@action
def manifest_job_iwd(test_dir):
    iwd = test_dir / "completed_manifest_job"
    iwd.mkdir()
    return iwd

@action
def completed_manifest_job(manifest_job_iwd, default_condor, test_job_hash):
    cmj = default_condor.submit({
        ** test_job_hash,
        "iwd": manifest_job_iwd,
        "manifest": "True"
        },
        count=1
    )
    assert cmj.wait(
        condition=ClusterState.all_terminal,
        timeout=60,
        verbose=True,
        fail_condition=ClusterState.any_held,
    )
    return cmj

@action
def manifest_dir_job_iwd(test_dir):
    iwd = test_dir / "completed_manifest_dir_job"
    iwd.mkdir()
    return iwd

@action
def completed_manifest_dir_job(manifest_dir_job_iwd, default_condor, test_job_hash):
    cmdj = default_condor.submit({
        ** test_job_hash,
        "iwd": manifest_dir_job_iwd,
        "manifest_dir": "custom-dir-name"
        },
        count=1
    )
    assert cmdj.wait(
        condition=ClusterState.all_terminal,
        timeout=60,
        verbose=True,
        fail_condition=ClusterState.any_held,
    )
    return cmdj

class TestManifest:
    def test_no_manifest(self, completed_no_manifest_job, no_manifest_job_iwd):
        manifest_dir_name = "{}_0_manifest".format(completed_no_manifest_job.clusterid)
        manifest_dir = no_manifest_job_iwd / manifest_dir_name
        assert manifest_dir not in no_manifest_job_iwd.iterdir()

    # parameterize this and the following test
    def test_manifest_contents(self, completed_manifest_job, manifest_job_iwd):
        manifest_dir_name = "{}_0_manifest".format(completed_manifest_job.clusterid)
        manifest_dir = manifest_job_iwd / manifest_dir_name
        assert manifest_dir in manifest_job_iwd.iterdir()

        file_names = [p.name for p in manifest_dir.iterdir()]
        assert "out" in file_names
        assert "in" in file_names
        assert "environment" in file_names

        with (manifest_dir / "in").open() as f:
            assert "input_file\n" in f.readlines()
        with (manifest_dir / "out").open() as f:
            assert "new_output_file\n" in f.readlines()

    def test_manifest_dir_specified(self, completed_manifest_dir_job, manifest_dir_job_iwd):
        manifest_dir_name = "custom-dir-name"
        manifest_dir = manifest_dir_job_iwd / manifest_dir_name
        assert manifest_dir in manifest_dir_job_iwd.iterdir()

        file_names = [p.name for p in manifest_dir.iterdir()]
        assert "out" in file_names
        assert "in" in file_names
        assert "environment" in file_names

        with (manifest_dir / "in").open() as f:
            assert "input_file\n" in f.readlines()
        with (manifest_dir / "out").open() as f:
            assert "new_output_file\n" in f.readlines()

<commit_msg>(#7381)  Refactor the tests to remove code duplication.<commit_after>#!/usr/bin/env pytest

from pathlib import Path

import logging

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

import htcondor

from ornithology import (
    action,
    config,
    Condor,
    ClusterState,
    JobStatus,
    format_script,
    write_file,
)

@action
def test_script_contents():
    return format_script(
        """
        #!/usr/bin/env python3

        with open("new_output_file", "w") as f:
            f.write("")
        exit(0)
        """
    )

@action
def test_script(test_dir, test_script_contents):
    test_script = test_dir / "test_script.py"
    write_file(test_script, test_script_contents)
    return test_script

@action
def test_job_hash(test_dir, path_to_python, test_script):
    input_file = test_dir / "input_file"
    input_file.touch()
    return {
        "executable": path_to_python,
        "arguments": test_script,
        "transfer_executable": "true",
        "should_transfer_files": "true",
        "transfer_input_files": input_file,
        "universe": "vanilla",
        "log": "log",
        "output": "output",
        "error": "error",
    }

TEST_PARAMETERS = {
    "no_manifest": {
        "iwd": "completed_no_manifest_job",
        "job_attrs": {},
        "manifest_dir_name": None,
    },
    "manifest": {
        "iwd": "completed_manifest_job",
        "job_attrs": {"manifest": "True",},
        "manifest_dir_name": "{}_0_manifest",
    },
    "manifest_dir": {
        "iwd": "completed_manifest_dir",
        "job_attrs": {"manifest_dir": "custom-dir-name",},
        "manifest_dir_name": "custom-dir-name",
    },
}

@action(params=TEST_PARAMETERS)
def test_parameters(request):
    return request.param

@action
def test_job_iwd(test_dir, test_parameters):
    iwd = test_dir / test_parameters["iwd"]
    iwd.mkdir()
    return iwd

@action
def completed_test_job(test_job_iwd, default_condor, test_job_hash, test_parameters):
    ctj = default_condor.submit(
        {**test_job_hash, **test_parameters["job_attrs"], "iwd": test_job_iwd,}, count=1
    )
    assert ctj.wait(
        condition=ClusterState.all_terminal,
        timeout=60,
        verbose=True,
        fail_condition=ClusterState.any_held,
    )
    return ctj

class TestManifest:
    def test_manifest_contents(self, completed_test_job, test_job_iwd, test_parameters):
        # This is a little clumsy, but probably better than duplicating
        # test_job_iwd and completed_test_job to avoid a conditional.
        if test_parameters["manifest_dir_name"] is None:
            manifest_dir_name = "{}_0_manifest".format(completed_test_job.clusterid)
            manifest_dir = test_job_iwd / manifest_dir_name
            assert manifest_dir not in test_job_iwd.iterdir()
            return

        manifest_dir_name = test_parameters["manifest_dir_name"].format(
            completed_test_job.clusterid
        )
        manifest_dir = test_job_iwd / manifest_dir_name
        assert manifest_dir in test_job_iwd.iterdir()

        file_names = [p.name for p in manifest_dir.iterdir()]
        assert "out" in file_names
        assert "in" in file_names
        assert "environment" in file_names

        with (manifest_dir / "in").open() as f:
            assert "input_file\n" in f.readlines()
        with (manifest_dir / "out").open() as f:
            assert "new_output_file\n" in f.readlines()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    data_reading_functions provides functionalities for data reading in the context of the wwdata package.
    Copyright (C) 2016 Chaim De Mulder

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
@authors: chaimdemulder, stijnvanhoey
contact: chaim.demulder@ugent.be
"""

import sys
import os
from os import listdir
import pandas as pd
import scipy as sp
import numpy as np
import matplotlib.pyplot as plt   #plotten in python
import xlrd

def list_files(path,ext):
    """

    """
    if ext == 'excel':
        files = [f for f in listdir(path) if '.xls' in f]
    elif ext == 'text':
        files = [f for f in listdir(path) if f.endswith('.txt')]
    elif ext == 'csv':
        files = [f for f in listdir(path) if f.endswith('.csv')]
    else:
        print('No files with',ext,'extension found in directory',path,'Please \
        choose one of the following: text, excel, csv')

        return None

    return files

def remove_empty_lines(path,ext):
    """
    """
    files = list_files(path,ext)
    if not files:
        print('Please provide a directory that contains '+ext+' files.')
        return None
    for filename in files:
        filepath = os.path.join(path,filename)
        data = pd.read_csv(filepath,sep='\t')
        data.dropna(axis=0,inplace=True)
        #filepath_new = os.path.join(path,filename+'_')
        data.to_csv(filepath,sep='\t',index=False,index_label=False)
    return None

def find_and_replace(path,ext,replace):
    """
    Finds the files with a certain extension in a directory and applies a find-
    replace action to those files. Removes the old files and produces files with
    a prefix stating the replacing value.

    Parameters
    ----------
    path : str
        the path name of the directory to apply the function to
    ext : str
        the extension of the files to be searched (excel, text or csv)
    replace : array of str
        the first value of replace is the string to be replaced by the second
        value of replace.
    """
    files = list_files(path,ext)
    if not files:
        print('Please provide a directory that contains '+ext+' files.')
        return None
    for filename in files:
        filepath = os.path.join(path,filename)
        filedata = None
        with open(filepath, 'r') as file :
            filedata = file.read()

        # Replace the target string
        filedata = filedata.replace(replace[0], replace[1])

        # Write the file out again
        with open(filepath, 'w') as file:
            file.write(filedata)

        #data = pd.read_csv(filepath,sep='\t')
        #data.replace(to_replace=replace[0],value=replace[1],inplace=True)
        #data.to_csv(filepath,sep='\t',index=False,index_label=False)

    return None

def sort_data(data,based_on,reset_index=[False,'new_index_name'],
              convert_to_timestamp=[True,'time_name','%d.%m.%Y %H:%M:%S']):
    """
    Sorts a dataset based on values in one of the columns and splits them in
    different dataframes, returned in the form of one dictionary

    Parameters
    ----------
    data : pd.dataframe
        the dataframe containing the data that needs to be sorted
    based_on : str
        the name of the column that contains the names or values the sorting
        should be based on
    reset_index : [bool,str]
        array indicating if the index of the sorted datasets should be reset to
        a new one; if first element is true, the second element is the title of
        the column to use as new index; default: False

    Returns
    -------
    dict :
        A dictionary of pandas dataframes with as labels those acquired from the
        based_on column
    """
    dictionary = {}
    measurement_codes = pd.Series(data[based_on].ravel()).unique()
    for i in measurement_codes:
        dictionary[i] = data[data[based_on]==i].drop(based_on,axis=1)
        if convert_to_timestamp[0] == True & reset_index[0] == True:
            dictionary[i][convert_to_timestamp[1]] = \
            pd.to_datetime(dictionary[i][convert_to_timestamp[1]],
                        format=convert_to_timestamp[2])
            dictionary[i].set_index(reset_index[1],inplace=True)
        elif convert_to_timestamp[0] == True & reset_index[0] == False:
            dictionary[i][convert_to_timestamp[1]] = \
            pd.to_datetime(dictionary[i][convert_to_timestamp[1]],
                        format=convert_to_timestamp[2])
        elif reset_index[0] == True & convert_to_timestamp[0] == False:
            dictionary[i].set_index(reset_index[1],inplace=True)
        print('Sorting',i,'...')

    return dictionary

def _get_header_length(read_file,ext='text',comment='#'):
    """
    Determines the amount of rows that are part of the header in a file that is
    already opened and readable

    Parameters
    ----------
    read_file : opened file
        an opened file object that is readable
    ext : str
        the extension (in words) of the file the headerlength needs to be found
        for
    comment : str
        comment symbol used in the files

    Returns
    -------
    headerlength : int
        the amount of rows that are part of the header in the read file

    """

    headerlength = 0
    header_test = comment
    counter = 0
    if ext == 'excel' or ext == 'zrx':
        while header_test == comment:
            header_test = str(read_file.sheet_by_index(0).cell_value(counter,0))[0]
            headerlength += 1
            counter +=1

    elif ext == 'text' or ext == 'csv':
        while header_test == comment:
            header_test = read_file.readline()[0]
            headerlength += 1

    return headerlength-1

def read_mat(path):
    """
    Reads in .mat datafiles and returns them as pd.DataFrame
    http://stackoverflow.com/questions/24762122/read-matlab-data-file-into-python-need-to-export-to-csv

    """

    #Also write separate script for converting all .mat files in one dir to .csv files

def _get_header_length(read_file,ext='text',comment='#'):
    """
    Determines the amount of rows that are part of the header in a file that is
    already opened and readable

    Parameters
    ----------
    read_file : opened file
        an opened file object that is readable
    ext : str
        the extension (in words) of the file the headerlength needs to be found
        for
    comment : str
        comment symbol used in the files

    Returns
    -------
    headerlength : int
        the amount of rows that are part of the header in the read file

    """

    headerlength = 0
    header_test = comment
    counter = 0
    if ext == 'excel' or ext == 'zrx':
        while header_test == comment:
            header_test = str(read_file.sheet_by_index(0).cell_value(counter,0))[0]
            headerlength += 1
            counter +=1

    elif ext == 'text':
        while header_test == comment:
            header_test = read_file.readline()[0]
            headerlength += 1

    return headerlength-1

def _open_file(filepath,ext='text'):
    """
    Opens file of a given extension in readable mode

    Parameters
    ----------
    filepath : str
        the complete path to the file to be opened in read mode
    ext : str
        the extension (in words) of the file that needs to be opened in read
        mode

    Returns
    -------
    The opened file in read mode

    """
    if ext == 'text' or ext == 'zrx' or ext == 'csv':
        return open(filepath, 'r')
    elif ext == 'excel':
        return xlrd.open_workbook(filepath)

def _read_file(filepath,ext='text',skiprows=0,sep='\t',encoding='utf8',decimal='.'):
    """
    Read a file of given extension and save it as a pandas dataframe

    Parameters
    ----------
    filepath : str
        the complete path to the file to be read and saved as dataframe
    ext : str
        the extension (in words) of the file that needs to be read and saved
    skiprows : int
        number of rows to skip when reading a file

    Returns
    -------
    A pandas dataframe containing the data from the given file

    """
    if ext == 'text':
        return pd.read_table(filepath,skiprows=skiprows,decimal='.',low_memory=False,index_col=None)
    elif ext == 'excel':
        return pd.read_excel(filepath,skiprows=skiprows,low_memory=False,index_col=None)
    elif ext == 'csv':
        return pd.read_csv(filepath,sep=sep,skiprows=skiprows,encoding=encoding,
			   error_bad_lines=False,low_memory=False,index_col=None)

def join_files(path,files,ext='text',sep=',',comment='#',encoding='utf8',decimal='.'):
    """
    Reads all files in a given directory, joins them and returns one pd.dataframe

    Parameters
    ----------
    path : str
	path to the folder that contains the files to be joined
    files : list
        list of files to be joined, must be the same extension
    ext : str
        extention of the files to read; possible: excel, text, csv
    sep : str
        the separating element (e.g. , or \t) necessary when reading csv-files
    comment : str
        comment symbol used in the files
    sort : array of bool and str
        if first element is true, apply the sort function to sort the data
        based on the tags in the column mentioned in the second element of the
        sort array

    Returns
    -------
    pd.dataframe:
        pandas dataframe containin concatenated files in the given directory
    """
    #Initialisations
    data = pd.DataFrame()

    #Select files based on extension and sort files alphabetically to make sure
    #they are added to each other in the correct order
    #files = list_files(path,ext)
    files.sort()
    print('joining',len(files),'files...')
    #Read files
    for file_name in files:
        dir_file_path = os.path.join(path,file_name)
        with _open_file(dir_file_path,ext) as read_file:
            headerlength = _get_header_length(read_file,ext,comment)
            data = data.append(_read_file(dir_file_path,ext=ext,sep=sep,
                                          skiprows=headerlength,
                                          decimal=decimal,encoding=encoding),
                                ignore_index=True)
        print('Adding file',file_name,'to dataframe')
    data.to_csv('joined_files',sep=sep)

    return data

def write_to_WEST(df,file_normal,file_west,units,filepath=os.getcwd()):
        """
        writes a text-file that is compatible with WEST. Adds the units as
        they are given in the 'units' argument.

        Parameters
        ----------
        df : pd.DataFrame
            the dataframe to write to WEST
        file_normal : str
            name of the original file to write, not yet compatible with WEST
        file_west : str
            name of the file that needs to be WEST compatible
        units : array of strings
            array containing the units for the respective columns in df
        filepath : str
            directory to save the files in; defaults to the current one

        Returns
        -------
        None; writes files
        """
        df.to_csv(os.path.join(filepath,file_normal),sep='\t')

        f = open(os.path.join(filepath,file_normal),'r')
        columns = f.readline()
        temp = f.read()
        f.close()

        f = open(os.path.join(filepath,file_west), 'w')
        f.write('#.t' + columns)
        unit_line = '#d\t'
        for i in range(0,len(units)-1):
            unit_line = unit_line + '{}\t'.format(units[i])
        unit_line = unit_line + '{}\n'.format(units[-1])
        f.write(unit_line)
        f.write(temp)
        f.close()
<commit_msg>add fillna when writing to west-readable file; can avoid problems with reading files in west<commit_after># -*- coding: utf-8 -*-
"""
    data_reading_functions provides functionalities for data reading in the context of the wwdata package.
    Copyright (C) 2016 Chaim De Mulder

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
@authors: chaimdemulder, stijnvanhoey
contact: chaim.demulder@ugent.be
"""

import sys
import os
from os import listdir
import pandas as pd
import scipy as sp
import numpy as np
import matplotlib.pyplot as plt   #plotten in python
import xlrd

def list_files(path,ext):
    """

    """
    if ext == 'excel':
        files = [f for f in listdir(path) if '.xls' in f]
    elif ext == 'text':
        files = [f for f in listdir(path) if f.endswith('.txt')]
    elif ext == 'csv':
        files = [f for f in listdir(path) if f.endswith('.csv')]
    else:
        print('No files with',ext,'extension found in directory',path,'Please \
        choose one of the following: text, excel, csv')

        return None

    return files

def remove_empty_lines(path,ext):
    """
    """
    files = list_files(path,ext)
    if not files:
        print('Please provide a directory that contains '+ext+' files.')
        return None
    for filename in files:
        filepath = os.path.join(path,filename)
        data = pd.read_csv(filepath,sep='\t')
        data.dropna(axis=0,inplace=True)
        #filepath_new = os.path.join(path,filename+'_')
        data.to_csv(filepath,sep='\t',index=False,index_label=False)
    return None

def find_and_replace(path,ext,replace):
    """
    Finds the files with a certain extension in a directory and applies a find-
    replace action to those files. Removes the old files and produces files with
    a prefix stating the replacing value.

    Parameters
    ----------
    path : str
        the path name of the directory to apply the function to
    ext : str
        the extension of the files to be searched (excel, text or csv)
    replace : array of str
        the first value of replace is the string to be replaced by the second
        value of replace.
    """
    files = list_files(path,ext)
    if not files:
        print('Please provide a directory that contains '+ext+' files.')
        return None
    for filename in files:
        filepath = os.path.join(path,filename)
        filedata = None
        with open(filepath, 'r') as file :
            filedata = file.read()

        # Replace the target string
        filedata = filedata.replace(replace[0], replace[1])

        # Write the file out again
        with open(filepath, 'w') as file:
            file.write(filedata)

        #data = pd.read_csv(filepath,sep='\t')
        #data.replace(to_replace=replace[0],value=replace[1],inplace=True)
        #data.to_csv(filepath,sep='\t',index=False,index_label=False)

    return None

def sort_data(data,based_on,reset_index=[False,'new_index_name'],
              convert_to_timestamp=[True,'time_name','%d.%m.%Y %H:%M:%S']):
    """
    Sorts a dataset based on values in one of the columns and splits them in
    different dataframes, returned in the form of one dictionary

    Parameters
    ----------
    data : pd.dataframe
        the dataframe containing the data that needs to be sorted
    based_on : str
        the name of the column that contains the names or values the sorting
        should be based on
    reset_index : [bool,str]
        array indicating if the index of the sorted datasets should be reset to
        a new one; if first element is true, the second element is the title of
        the column to use as new index; default: False

    Returns
    -------
    dict :
        A dictionary of pandas dataframes with as labels those acquired from the
        based_on column
    """
    dictionary = {}
    measurement_codes = pd.Series(data[based_on].ravel()).unique()
    for i in measurement_codes:
        dictionary[i] = data[data[based_on]==i].drop(based_on,axis=1)
        if convert_to_timestamp[0] == True & reset_index[0] == True:
            dictionary[i][convert_to_timestamp[1]] = \
            pd.to_datetime(dictionary[i][convert_to_timestamp[1]],
                        format=convert_to_timestamp[2])
            dictionary[i].set_index(reset_index[1],inplace=True)
        elif convert_to_timestamp[0] == True & reset_index[0] == False:
            dictionary[i][convert_to_timestamp[1]] = \
            pd.to_datetime(dictionary[i][convert_to_timestamp[1]],
                        format=convert_to_timestamp[2])
        elif reset_index[0] == True & convert_to_timestamp[0] == False:
            dictionary[i].set_index(reset_index[1],inplace=True)
        print('Sorting',i,'...')

    return dictionary

def _get_header_length(read_file,ext='text',comment='#'):
    """
    Determines the amount of rows that are part of the header in a file that is
    already opened and readable

    Parameters
    ----------
    read_file : opened file
        an opened file object that is readable
    ext : str
        the extension (in words) of the file the headerlength needs to be found
        for
    comment : str
        comment symbol used in the files

    Returns
    -------
    headerlength : int
        the amount of rows that are part of the header in the read file

    """

    headerlength = 0
    header_test = comment
    counter = 0
    if ext == 'excel' or ext == 'zrx':
        while header_test == comment:
            header_test = str(read_file.sheet_by_index(0).cell_value(counter,0))[0]
            headerlength += 1
            counter +=1

    elif ext == 'text' or ext == 'csv':
        while header_test == comment:
            header_test = read_file.readline()[0]
            headerlength += 1

    return headerlength-1

def read_mat(path):
    """
    Reads in .mat datafiles and returns them as pd.DataFrame
    http://stackoverflow.com/questions/24762122/read-matlab-data-file-into-python-need-to-export-to-csv

    """

    #Also write separate script for converting all .mat files in one dir to .csv files

def _get_header_length(read_file,ext='text',comment='#'):
    """
    Determines the amount of rows that are part of the header in a file that is
    already opened and readable

    Parameters
    ----------
    read_file : opened file
        an opened file object that is readable
    ext : str
        the extension (in words) of the file the headerlength needs to be found
        for
    comment : str
        comment symbol used in the files

    Returns
    -------
    headerlength : int
        the amount of rows that are part of the header in the read file

    """

    headerlength = 0
    header_test = comment
    counter = 0
    if ext == 'excel' or ext == 'zrx':
        while header_test == comment:
            header_test = str(read_file.sheet_by_index(0).cell_value(counter,0))[0]
            headerlength += 1
            counter +=1

    elif ext == 'text':
        while header_test == comment:
            header_test = read_file.readline()[0]
            headerlength += 1

    return headerlength-1

def _open_file(filepath,ext='text'):
    """
    Opens file of a given extension in readable mode

    Parameters
    ----------
    filepath : str
        the complete path to the file to be opened in read mode
    ext : str
        the extension (in words) of the file that needs to be opened in read
        mode

    Returns
    -------
    The opened file in read mode

    """
    if ext == 'text' or ext == 'zrx' or ext == 'csv':
        return open(filepath, 'r')
    elif ext == 'excel':
        return xlrd.open_workbook(filepath)

def _read_file(filepath,ext='text',skiprows=0,sep='\t',encoding='utf8',decimal='.'):
    """
    Read a file of given extension and save it as a pandas dataframe

    Parameters
    ----------
    filepath : str
        the complete path to the file to be read and saved as dataframe
    ext : str
        the extension (in words) of the file that needs to be read and saved
    skiprows : int
        number of rows to skip when reading a file

    Returns
    -------
    A pandas dataframe containing the data from the given file

    """
    if ext == 'text':
        return pd.read_table(filepath,skiprows=skiprows,decimal='.',low_memory=False,index_col=None)
    elif ext == 'excel':
        return pd.read_excel(filepath,skiprows=skiprows,low_memory=False,index_col=None)
    elif ext == 'csv':
        return pd.read_csv(filepath,sep=sep,skiprows=skiprows,encoding=encoding,
                           error_bad_lines=False,low_memory=False,index_col=None)

def join_files(path,files,ext='text',sep=',',comment='#',encoding='utf8',decimal='.'):
    """
    Reads all files in a given directory, joins them and returns one pd.dataframe

    Parameters
    ----------
    path : str
	path to the folder that contains the files to be joined
    files : list
        list of files to be joined, must be the same extension
    ext : str
        extention of the files to read; possible: excel, text, csv
    sep : str
        the separating element (e.g. , or \t) necessary when reading csv-files
    comment : str
        comment symbol used in the files
    sort : array of bool and str
        if first element is true, apply the sort function to sort the data
        based on the tags in the column mentioned in the second element of the
        sort array

    Returns
    -------
    pd.dataframe:
        pandas dataframe containin concatenated files in the given directory
    """
    #Initialisations
    data = pd.DataFrame()

    #Select files based on extension and sort files alphabetically to make sure
    #they are added to each other in the correct order
    #files = list_files(path,ext)
    files.sort()
    print('joining',len(files),'files...')
    #Read files
    for file_name in files:
        dir_file_path = os.path.join(path,file_name)
        with _open_file(dir_file_path,ext) as read_file:
            headerlength = _get_header_length(read_file,ext,comment)
            data = data.append(_read_file(dir_file_path,ext=ext,sep=sep,
                                          skiprows=headerlength,
                                          decimal=decimal,encoding=encoding),
                                ignore_index=True)
        print('Adding file',file_name,'to dataframe')
    data.to_csv('joined_files',sep=sep)

    return data

def write_to_WEST(df,file_normal,file_west,units,filepath=os.getcwd(),fillna=True):
        """
        writes a text-file that is compatible with WEST. Adds the units as
        they are given in the 'units' argument.

        Parameters
        ----------
        df : pd.DataFrame
            the dataframe to write to WEST
        file_normal : str
            name of the original file to write, not yet compatible with WEST
        file_west : str
            name of the file that needs to be WEST compatible
        units : array of strings
            array containing the units for the respective columns in df
        filepath : str
            directory to save the files in; defaults to the current one
        fillna : bool
            when True, replaces nan values with 0 values (this might avoid
            WEST problems later one).

        Returns
        -------
        None; writes files
        """
        if fillna:
            df = df.fillna(0)
        df.to_csv(os.path.join(filepath,file_normal),sep='\t')

        f = open(os.path.join(filepath,file_normal),'r')
        columns = f.readline()
        temp = f.read()
        f.close()

        f = open(os.path.join(filepath,file_west), 'w')
        f.write('#.t' + columns)
        unit_line = '#d\t'
        for i in range(0,len(units)-1):
            unit_line = unit_line + '{}\t'.format(units[i])
        unit_line = unit_line + '{}\n'.format(units[-1])
        f.write(unit_line)
        f.write(temp)
        f.close()
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from odoo.tests import common

class TestCodesModelsRelation(common.TransactionCase):

    def setUp(self):
        super(TestCodesModelsRelation, self).setUp()
        country_ar = self.env.ref('base.ar')
        self.test_codes_models_country = self.env['codes.models.relation'].create({
            'name': 'test',
            'name_model': 'res.country',
            'id_model': country_ar.id,
            'code': 'x'
        })

    def test_model_attribute(self):
        self.assertEqual(self.test_codes_models_country.get_record().code, 'AR')

    def test_invalid_model(self):
        self.test_codes_models_country.name_model = 'res.countri'
        with self.assertRaises(Exception):
            self.test_codes_models_country.get_record()

    def test_invalid_record_id(self):
        self.test_codes_models_country.id_model = -1
        with self.assertRaises(Exception):
            record = self.test_codes_models_country.get_record()
            record.name

    def test_unique(self):
        country_ar = self.env.ref('base.ar')
        with self.assertRaises(Exception):
            #IntegrityError
            self.test_codes_models_country2 = self.env['codes.models.relation'].create({
                'name': 'test',
                'name_model': 'res.country',
                'id_model': country_ar.id,
                'code': 'x'
            })

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:<commit_msg>[UPD] Tests<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from odoo.tests import common
from openerp.exceptions import Warning

class TestCodesModelsRelation(common.TransactionCase):

    def setUp(self):
        super(TestCodesModelsRelation, self).setUp()
        country_ar = self.env.ref('base.ar')
        self.test_codes_models_country = self.env['codes.models.relation'].create({
            'name': 'test',
            'name_model': 'res.country',
            'id_model': country_ar.id,
            'code': 'x'
        })

    def test_model_attribute(self):
        self.assertEqual(self.test_codes_models_country.get_record().code, 'AR')

    def test_invalid_model(self):
        self.test_codes_models_country.name_model = 'res.countri'
        with self.assertRaises(Exception):
            self.test_codes_models_country.get_record()

    def test_invalid_record_id(self):
        self.test_codes_models_country.id_model = -1
        with self.assertRaises(Warning):
            record = self.test_codes_models_country.get_record()
            record.name

    def test_unique(self):
        country_ar = self.env.ref('base.ar')
        with self.assertRaises(Exception):
            #IntegrityError
            self.test_codes_models_country2 = self.env['codes.models.relation'].create({
                'name': 'test',
                'name_model': 'res.country',
                'id_model': country_ar.id,
                'code': 'x'
            })

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:<|endoftext|>


<commit_before>#! /usr/bin/env python3

from subprocess import *
import sys
from os import path

sys.path.append(path.sep.join(path.abspath(__file__).split(path.sep)[:-3] + ['src']))

import grapher
import bezier

def graph(name, function):
    grapher.main(name, function[0])

def bezier(name, p_list):
    bezier.main(name, p_list)
<commit_msg>commit<commit_after>#! /usr/bin/env python3

from subprocess import *
import sys
from os import path

sys.path.append(path.sep.join(path.abspath(__file__).split(path.sep)[:-3] + ['src']))

import grapher
import bezier as bz

def graph(name, function):
    grapher.main(name, function[0])

def bezier(name, p_list):
    bz.main(name, p_list)
<|endoftext|>


<commit_before><commit_msg>Moved sine wave generation inside TimeSeries. Moved rectangle window inside PowerSpectrum.<commit_after><|endoftext|>


<commit_before>"""
cryptography.py
Author: Anoushka Alavilli
Credit: Mr. Dennison, Sarah Dunbar

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
message= input("Please enter a message. ")
key= input("Please enter a key. ")
listnumsofmessage= []
listnumsofkey= []

#ENCRYPTION:
for i in (message):
    listnumsofmessage.append(associations.find(i))
print (listnumsofmessage)

for i in (key):
    listnumsofkey.append(associations.find(i))
print (listnumsofkey)

keylength = len(key)
messagelength = len(message)
increasefactor = 1 + messagelength//keylength
key= increasefactor*key
print (message, key)
sumval= []
for i in (range(messagelength)):
    print (message[i], key[i])

for i in (key):
    listnumsofkey.append(associations.find(i))
print ((listnumsofmessage), (listnumsofkey))

numbersinmessage= (len(listnumsofmessage))
print (numbersinmessage)
encrypted= []
for i in range (numbersinmessage):
    placeinassociations= ((listnumsofmessage[i]) + (listnumsofkey[i]))
    encrypted.append(associations[placeinassociations])
print (encrypted)

#DECRYPTION:
for i in (encrypted):

    #print (associations.find(message[i], key[i]))

#print (((listnumsofmessage).count(i))%((listnumsofkey).count(i)))
    #for x in (listnumsofmessage):
        #for y in (listnumsofkey):
            #print (x,y)
        #break

#lenkey= (len(listnumsofkey))
#rangekey= (range(lenkey))
#print (rangekey)
#iinrangekey= (i in rangekey)
#for x in range(len(listnumsofmessage)):
   #print (x, iinrangekey)

#x= lambda ((len(listnumsofmessage))*x)%(len(listnumsofkey))=0)

#if (len(listnumsofmessage)) >= (len(listnumsofkey)):
 #   ((listnumsofkey)*x)<commit_msg>Updated from Brython Server: 11/3/2015 9:24:16 PM<commit_after>"""
cryptography.py
Author: Anoushka Alavilli
Credit: Mr. Dennison, Sarah Dunbar

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"
message= input("Please enter a message. ")
key= input("Please enter a key. ")
listnumsofmessage= []
listnumsofkey= []

#ENCRYPTION:
for i in (message):
    listnumsofmessage.append(associations.find(i))
print (listnumsofmessage)

for i in (key):
    listnumsofkey.append(associations.find(i))
print (listnumsofkey)

keylength = len(key)
messagelength = len(message)
increasefactor = 1 + messagelength//keylength
key= increasefactor*key
print (message, key)
sumval= []
for i in (range(messagelength)):
    print (message[i], key[i])

for i in (key):
    listnumsofkey.append(associations.find(i))
print ((listnumsofmessage), (listnumsofkey))

numbersinmessage= (len(listnumsofmessage))
print (numbersinmessage)
encrypted= []
for i in range (numbersinmessage):
    placeinassociations= ((listnumsofmessage[i]) + (listnumsofkey[i]))
    encrypted.append(associations[placeinassociations])
print (encrypted)

#DECRYPTION:
for i in (encrypted):
    print ("hello")

    #print (associations.find(message[i], key[i]))

#print (((listnumsofmessage).count(i))%((listnumsofkey).count(i)))
    #for x in (listnumsofmessage):
        #for y in (listnumsofkey):
            #print (x,y)
        #break

#lenkey= (len(listnumsofkey))
#rangekey= (range(lenkey))
#print (rangekey)
#iinrangekey= (i in rangekey)
#for x in range(len(listnumsofmessage)):
   #print (x, iinrangekey)

#x= lambda ((len(listnumsofmessage))*x)%(len(listnumsofkey))=0)

#if (len(listnumsofmessage)) >= (len(listnumsofkey)):
 #   ((listnumsofkey)*x)<|endoftext|>


<commit_before>#!/usr/bin/python

import os
import plistlib
import sys
from pprint import pprint
from xml.parsers.expat import ExpatError
from distutils import version
from types import StringType

MUNKI_ROOT_PATH = '/Users/Shared/munki_repo'
###### Where is the path to your Munki repo?
###### Uncomment the line below if your Munki repo is somewhere else
# MUNKI_ROOT_PATH = 'Put/Path/You/Want'

# Where should old packages be moved to?
# Guess at user trash
where_to_dump=os.path.join(os.getenv("HOME"), ".Trash")

###### Uncomment the line below if you'd like the packages to go somewhere other than the trash folder
# where_to_dump='/Put/Path/You/Want'

# Where is make catalogs?
makecatalogs='/usr/local/munki/makecatalogs'

# Double-check the user trash exists
if not os.path.isdir(where_to_dump):
	# If the directory doesn't already exist, make it
	where_to_dump=os.makedirs(where_to_dump)

MUNKI_PKGS_DIR_NAME = 'pkgs'
MUNKI_PKGSINFO_DIR_NAME = 'pkgsinfo'

# Stolen from Munki's munkicommon.py
class MunkiLooseVersion(version.LooseVersion):
    '''Subclass version.LooseVersion to compare things like
    "10.6" and "10.6.0" as equal'''

    def __init__(self, vstring=None):
        if vstring is None:
            # treat None like an empty string
            self.parse('')
        if vstring is not None:
            if isinstance(vstring, unicode):
                # unicode string! Why? Oh well...
                # convert to string so version.LooseVersion doesn't choke
                vstring = vstring.encode('UTF-8')
            self.parse(str(vstring))

    def _pad(self, version_list, max_length):
        """Pad a version list by adding extra 0
        components to the end if needed"""
        # copy the version_list so we don't modify it
        cmp_list = list(version_list)
        while len(cmp_list) < max_length:
            cmp_list.append(0)
        return cmp_list

    def __cmp__(self, other):
        if isinstance(other, StringType):
            other = MunkiLooseVersion(other)

        max_length = max(len(self.version), len(other.version))
        self_cmp_version = self._pad(self.version, max_length)
        other_cmp_version = self._pad(other.version, max_length)

        return cmp(self_cmp_version, other_cmp_version)

# Function that moves from old location to new location for a list of items
def trash_old_stuff(trashlist, newpath):
	if isinstance(trashlist, list):
		for old_location in trashlist:
			# Get just the filename from the full path
			path_within_destination=os.path.relpath(old_location, trashpath)
			new_location=os.path.join(newpath, path_within_destination)
			if not os.path.exists(os.path.dirname(new_location)):
				os.makedirs(os.path.dirname(new_location))
			if os.path.exists(old_location) and os.path.isdir(newpath):
				os.rename(old_location, new_location)
				print "Moving %s to %s\n" % (old_location, new_location)
			else:
				print "One of %s or %s does not exist\n" % (old_location, new_location)
	else:
		print "%s is not a valid list\n" % trashlist

# Join paths based on what's defined
pkgsinfo_path=os.path.join(MUNKI_ROOT_PATH, MUNKI_PKGSINFO_DIR_NAME)
pkgs_path=os.path.join(MUNKI_ROOT_PATH, MUNKI_PKGS_DIR_NAME)

# Check that the paths for the pkgsinfo and pkgs exist
if not os.path.isdir(pkgsinfo_path) and not os.path.isdir(pkgs_path):
	print "Your pkgsinfo and pkgs paths ae not valid. Please check your MUNKI_ROOT_PATH value"
else:

	# A list to store all items
	all_items = {};

	# Lists to store items to delete
	pkgs_to_delete = [];
	pkgsinfo_to_delete = [];

	# Walk through the pkgsinfo files...
	for root, dirs, files in os.walk(pkgsinfo_path):
		for dir in dirs:
			# Skip directories starting with a period
			if dir.startswith("."):
				dirs.remove(dir)
		for file in files:
			# Skip files that start with a period
			if file.startswith("."):
				continue
			fullfile = os.path.join(root, file)
			plist = plistlib.readPlist(fullfile)
			plistname = plist['name']
			plistversion = plist['version']
			plistcatalogs = plist['catalogs']
			plistcatalogs.sort()
			# Some items won't have an installer_item_location: nopkg .plist files, for example... that's okay
			if 'installer_item_location' in plist:
				plistinstaller_item_location = os.path.join(pkgs_path, plist['installer_item_location'])
			else:
				plistinstaller_item_location = ''

			# Create a dictionary based on the plist values read
			plistdict={ 'pkginfo': fullfile, 'version': plistversion, 'catalogs': plistcatalogs, 'installer_item_location': plistinstaller_item_location}

			# See if the plist name is already in all_items
			if plistname in all_items:
				# Compare the previously existing one to the currently focused one to see if they have the same catalogs (fix this because it could be testing production or production testing)
				if all_items[plistname]['catalogs'] == plistcatalogs:
					# See if this is a newer version than the one in there
					if cmp (MunkiLooseVersion(plistversion), MunkiLooseVersion(all_items[plistname]['version'])) > 0 :
						# If this is newer, then move the old one to the items to delete list
						if( all_items[plistname]['installer_item_location'] != '' ):
							pkgs_to_delete.append(all_items[plistname]['installer_item_location'])
						pkgsinfo_to_delete.append(all_items[plistname]['pkginfo'])
						del all_items[plistname]
						all_items[plistname]=plistdict
					else:
						# Otherwise, if this is older, keep the old one in there, and move this one to the delete list
						if( plistdict['installer_item_location'] != '' ):
							pkgs_to_delete.append(plistdict['installer_item_location'])
						pkgsinfo_to_delete.append(plistdict['pkginfo'])
			else:
				# If it's not in the list already, add it
				all_items[plistname]=plistdict

	trash_old_stuff(pkgs_to_delete, where_to_dump)
	trash_old_stuff(pkgsinfo_to_delete, where_to_dump)

	# If /usr/local/munki/makecatalogs exists (it should), then run it to reflect the changes or let the user know to run it
	if os.path.exists(makecatalogs):
		print "Running makecatalogs"
		os.system(makecatalogs)
	else:
		print "%s could not be found. When you have a chance, run makecatalogs on your Munki repo to have the changes reflected." % makecatalogs
<commit_msg>Added in trashpaths<commit_after>#!/usr/bin/python

import os
import plistlib
import sys
from pprint import pprint
from xml.parsers.expat import ExpatError
from distutils import version
from types import StringType

MUNKI_ROOT_PATH = '/Users/Shared/munki_repo'
###### Where is the path to your Munki repo?
###### Uncomment the line below if your Munki repo is somewhere else
# MUNKI_ROOT_PATH = 'Put/Path/You/Want'

# Where should old packages be moved to?
# Guess at user trash
where_to_dump=os.path.join(os.getenv("HOME"), ".Trash")

###### Uncomment the line below if you'd like the packages to go somewhere other than the trash folder
# where_to_dump='/Put/Path/You/Want'

# Where is make catalogs?
makecatalogs='/usr/local/munki/makecatalogs'

# Double-check the user trash exists
if not os.path.isdir(where_to_dump):
	# If the directory doesn't already exist, make it
	where_to_dump=os.makedirs(where_to_dump)

MUNKI_PKGS_DIR_NAME = 'pkgs'
MUNKI_PKGSINFO_DIR_NAME = 'pkgsinfo'

# Stolen from Munki's munkicommon.py
class MunkiLooseVersion(version.LooseVersion):
    '''Subclass version.LooseVersion to compare things like
    "10.6" and "10.6.0" as equal'''

    def __init__(self, vstring=None):
        if vstring is None:
            # treat None like an empty string
            self.parse('')
        if vstring is not None:
            if isinstance(vstring, unicode):
                # unicode string! Why? Oh well...
                # convert to string so version.LooseVersion doesn't choke
                vstring = vstring.encode('UTF-8')
            self.parse(str(vstring))

    def _pad(self, version_list, max_length):
        """Pad a version list by adding extra 0
        components to the end if needed"""
        # copy the version_list so we don't modify it
        cmp_list = list(version_list)
        while len(cmp_list) < max_length:
            cmp_list.append(0)
        return cmp_list

    def __cmp__(self, other):
        if isinstance(other, StringType):
            other = MunkiLooseVersion(other)

        max_length = max(len(self.version), len(other.version))
        self_cmp_version = self._pad(self.version, max_length)
        other_cmp_version = self._pad(other.version, max_length)

        return cmp(self_cmp_version, other_cmp_version)

# Function that moves from old location to new location for a list of items
def trash_old_stuff(trashlist, trashpath, newpath):
	if isinstance(trashlist, list):
		for old_location in trashlist:
			# Get just the filename from the full path
			path_within_destination=os.path.relpath(old_location, trashpath)
			new_location=os.path.join(newpath, path_within_destination)
			if not os.path.exists(os.path.dirname(new_location)):
				os.makedirs(os.path.dirname(new_location))
			if os.path.exists(old_location) and os.path.isdir(newpath):
				os.rename(old_location, new_location)
				print "Moving %s to %s\n" % (old_location, new_location)
			else:
				print "One of %s or %s does not exist\n" % (old_location, new_location)
	else:
		print "%s is not a valid list\n" % trashlist

# Join paths based on what's defined
pkgsinfo_path=os.path.join(MUNKI_ROOT_PATH, MUNKI_PKGSINFO_DIR_NAME)
pkgs_path=os.path.join(MUNKI_ROOT_PATH, MUNKI_PKGS_DIR_NAME)

# Check that the paths for the pkgsinfo and pkgs exist
if not os.path.isdir(pkgsinfo_path) and not os.path.isdir(pkgs_path):
	print "Your pkgsinfo and pkgs paths ae not valid. Please check your MUNKI_ROOT_PATH value"
else:

	# A list to store all items
	all_items = {};

	# Lists to store items to delete
	pkgs_to_delete = [];
	pkgsinfo_to_delete = [];

	# Walk through the pkgsinfo files...
	for root, dirs, files in os.walk(pkgsinfo_path):
		for dir in dirs:
			# Skip directories starting with a period
			if dir.startswith("."):
				dirs.remove(dir)
		for file in files:
			# Skip files that start with a period
			if file.startswith("."):
				continue
			fullfile = os.path.join(root, file)
			plist = plistlib.readPlist(fullfile)
			plistname = plist['name']
			plistversion = plist['version']
			plistcatalogs = plist['catalogs']
			plistcatalogs.sort()
			# Some items won't have an installer_item_location: nopkg .plist files, for example... that's okay
			if 'installer_item_location' in plist:
				plistinstaller_item_location = os.path.join(pkgs_path, plist['installer_item_location'])
			else:
				plistinstaller_item_location = ''

			# Create a dictionary based on the plist values read
			plistdict={ 'pkginfo': fullfile, 'version': plistversion, 'catalogs': plistcatalogs, 'installer_item_location': plistinstaller_item_location}

			# See if the plist name is already in all_items
			if plistname in all_items:
				# Compare the previously existing one to the currently focused one to see if they have the same catalogs (fix this because it could be testing production or production testing)
				if all_items[plistname]['catalogs'] == plistcatalogs:
					# See if this is a newer version than the one in there
					if cmp (MunkiLooseVersion(plistversion), MunkiLooseVersion(all_items[plistname]['version'])) > 0 :
						# If this is newer, then move the old one to the items to delete list
						if( all_items[plistname]['installer_item_location'] != '' ):
							pkgs_to_delete.append(all_items[plistname]['installer_item_location'])
						pkgsinfo_to_delete.append(all_items[plistname]['pkginfo'])
						del all_items[plistname]
						all_items[plistname]=plistdict
					else:
						# Otherwise, if this is older, keep the old one in there, and move this one to the delete list
						if( plistdict['installer_item_location'] != '' ):
							pkgs_to_delete.append(plistdict['installer_item_location'])
						pkgsinfo_to_delete.append(plistdict['pkginfo'])
			else:
				# If it's not in the list already, add it
				all_items[plistname]=plistdict

	trash_old_stuff(pkgs_to_delete, pkgs_path, where_to_dump)
	trash_old_stuff(pkgsinfo_to_delete, pkgsinfo_path, where_to_dump)

	# If /usr/local/munki/makecatalogs exists (it should), then run it to reflect the changes or let the user know to run it
	if os.path.exists(makecatalogs):
		print "Running makecatalogs"
		os.system(makecatalogs)
	else:
		print "%s could not be found. When you have a chance, run makecatalogs on your Munki repo to have the changes reflected." % makecatalogs
<|endoftext|>


<commit_before>"""
cryptography.py
Author: Andreas Moberg
Credit: Payton (for mad math skillz), Ashwini Chauswerry, UncleZiev

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"

#print(len(associations))
#85

keystroke=input("Enter e to encrypt, d to decrypt, or q to quit:")

quit=False

#if keystroke != "e" or "d" or "q":
    #print("Did not understand command, try again.")

if keystroke=="e":
    mess=input("Message:")
    key=input("Key:")
    k=len(key)
    m=len(mess)
    cryptm=[]
    cryptk=[]
    encrypt=[]
    tencrypt=[]

    if m>k:
        count=key*int((m-(m%k))/k)
        trun=key[0:(m%k)]
        newkey=count+trun
    elif k>m:
        newkey=key[0:m]
        print(newkey, message)

    for x in mess:
        cryptm.append(associations.find(x))
    for x in newkey:
        cryptk.append(associations.find(x))

    encrypt=[sum(x) for x in zip(cryptm, cryptk)]

    encrypt=[x-85 for x in encrypt]

#    for x in encrypt:
#       if x>85:
#            x.replace(x, x-85)

    for x in encrypt:
        tencrypt.append(associations[x])
    for x in tencrypt:
        print(x, end="")

if keystroke=="d":
    mess=input("Message:")
    key=input("Key:")
    k=len(key)
    m=len(mess)
    mes=[]
    cryptm=[]
    cryptk=[]
    decrypt=[]
    tdecrypt=[]

    if m>k:
        count=key*int((m-(m%k))/k)
        trun=key[0:(m%k)]
        newkey=count+trun
    elif k>m:
        newkey=key[0:m]
        print(newkey, message)

    for x in mess:
        cryptm.append(associations.find(x))
    for x in newkey:
        cryptk.append(associations.find(x))

    decrypt=(x-y for x, y in zip(cryptm, cryptk))

    decrypt=[x+85 for x in decrypt]

    for x in decrypt:
        tdecrypt.append(associations[x])
    for x in tdecrypt:
        print(x, end="")

if keystroke=="q":
    quit=True
    print("Goodbye!")

<commit_msg>Updated from Brython Server: 4/20/2016 9:29:58 PM<commit_after>"""
cryptography.py
Author: Andreas Moberg
Credit: Payton (for mad math skillz), Ashwini Chauswerry, UncleZiev

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!"

#print(len(associations))
#85

keystroke=input("Enter e to encrypt, d to decrypt, or q to quit:")

quit=False

#if keystroke != "e" or "d" or "q":
    #print("Did not understand command, try again.")

if keystroke=="e":
    mess=input("Message:")
    key=input("Key:")
    k=len(key)
    m=len(mess)
    cryptm=[]
    cryptk=[]
    encrypt=[]
    tencrypt=[]

    if m>k:
        count=key*int((m-(m%k))/k)
        trun=key[0:(m%k)]
        newkey=count+trun
    elif k>m:
        newkey=key[0:m]
        print(newkey, message)

    for x in mess:
        cryptm.append(associations.find(x))
    for x in newkey:
        cryptk.append(associations.find(x))

    encrypt=[sum(x) for x in zip(cryptm, cryptk)]
    for x in encrypt:
        if x>85:
            encrypt=[x-85 for x in encrypt]
        if x<85

#    for x in encrypt:
#       if x>85:
#            x.replace(x, x-85)

    for x in encrypt:
        tencrypt.append(associations[x])
    for x in tencrypt:
        print(x, end="")

if keystroke=="d":
    mess=input("Message:")
    key=input("Key:")
    k=len(key)
    m=len(mess)
    mes=[]
    cryptm=[]
    cryptk=[]
    decrypt=[]
    tdecrypt=[]

    if m>k:
        count=key*int((m-(m%k))/k)
        trun=key[0:(m%k)]
        newkey=count+trun
    elif k>m:
        newkey=key[0:m]
        print(newkey, message)

    for x in mess:
        cryptm.append(associations.find(x))
    for x in newkey:
        cryptk.append(associations.find(x))

    decrypt=(x-y for x, y in zip(cryptm, cryptk))

    decrypt=[x-85 for x in decrypt]

    for x in decrypt:
        tdecrypt.append(associations[x])
    for x in tdecrypt:
        print(x, end="")

if keystroke=="q":
    quit=True
    print("Goodbye!")

<|endoftext|>


<commit_before>"""
upload_manifest.py

Uploads the files in a manifest to Amazon Glacier.

"""
import os
import sys
import time
from Furtive import Furtive

try:
        import boto
	import boto.glacier
	import boto.glacier.layer2
except ImportError:
	print "boto must be installed to use this script. Run one of the commands"
	print " below to install boto:\n"
	print "\"easy_install boto\""
	print "\"pip install boto\""
        raise

try:
    import argparse
except ImportError:
    print "argparse must be installed manually for Python 2.6 and earlier."
    print "please download argparse from http://pypi.python.org/pypi/argparse\n"
    print "Installation can also be attempted with:"
    print "\"easy_install argparse\""
    print "\"pip install argparse\""
    raise

def main():
    start_time = time.time()
    old_cwd = os.getcwd()

    region_names = ""
    for i in boto.glacier.regions():
        region_names = region_names + i.name + " "

    parser = argparse.ArgumentParser(description='Upload a manifest to Amazon Glacier or S3')
    parser.add_argument('--dir', action="store", default=".",
                         help='''Directory containing the manifest. Default: .''')
    parser.add_argument('--aws-secret-access-key', action="store", required=True, help="Your AWS Secret Key")
    parser.add_argument('--aws-access-key-id', action="store", required=True, help="Your AWS Access Key ID")
    parser.add_argument('--vault-name', action="store", required=True, help="Name of the new vault")
    parser.add_argument('--region', action="store", required=True, help="""Glacier region to connect to.
    	                 Available regions: """ + region_names
    	                )
    parser.add_argument('--show-progress',action="store_true",
                        default=False, help='''When this flag is present,
                                               a progress indicator will
                                               show on STDOUT.
                                               Default: False''')
    parser.add_argument('--manifest', action="store", dest="manifest",
    	                default='.manifest.db',
                        help='''Location of the manifast file. Manifests may
                                be located outside the directory indicated by
                                --dir. Must provide path and filename of
                                the manifest file. Default: DIR/.manifest.db''')
    args = parser.parse_args()

    layer2 = boto.glacier.layer2.Layer2(aws_access_key_id=args.aws_access_key_id,
                                        aws_secret_access_key=args.aws_secret_access_key,
                                        region_name=args.region)

    try:
        fur = Furtive(args.dir)
    except:
        raise
    fur.set_manifest(args.manifest)

    try:
        hashes = fur.get_previous_hashes()
    except:
        raise

    if len(hashes) == 0:
        print "No files in manifest. Perhaps you should run hashDir.py to create a manifest first."
        raise Exception("no_manifest")

    vault = layer2.create_vault(args.vault_name)

    file_num = 0
    total_num_files = len(hashes)
    os.chdir(args.dir)

    for file in hashes.keys():
        if args.show_progress:
            file_num = file_num + 1
            print "\n",file, os.path.getsize(file)
            progress = round((float(file_num) / total_num_files) * 100,1)
            sys.stdout.write("\r" + str(progress) + "% " +
                             str(file_num) + " of " + str(total_num_files))
            sys.stdout.flush()
        attempt = 1
        while attempt < 5:
            try:
                vault.upload_archive(file)
            except UnexpectedHTTPResponseError, e:
                attempt = attempt + 1
                print "Error Uploading %s. Retry %s. Error Deatils %s" % (file,str(attempt),e)
                continue
            break

    #Upload Manifest DB
    vault.upload_archive(fur.manifest_file)

    end_time = time.time()

    if args.show_progress:
        print "\n"

    print "Finished uploading files to the vault %s" % (args.vault_name)
    print "Time taken: %s seconds" % (str(end_time - start_time))

if __name__ == "__main__":
    main()
<commit_msg>Added more reporting for failed uploads<commit_after>"""
upload_manifest.py

Uploads the files in a manifest to Amazon Glacier.

"""
import os
import sys
import time
from Furtive import Furtive

try:
        import boto
	import boto.glacier
	import boto.glacier.layer2
except ImportError:
	print "boto must be installed to use this script. Run one of the commands"
	print " below to install boto:\n"
	print "\"easy_install boto\""
	print "\"pip install boto\""
        raise

try:
    import argparse
except ImportError:
    print "argparse must be installed manually for Python 2.6 and earlier."
    print "please download argparse from http://pypi.python.org/pypi/argparse\n"
    print "Installation can also be attempted with:"
    print "\"easy_install argparse\""
    print "\"pip install argparse\""
    raise

def main():
    start_time = time.time()
    old_cwd = os.getcwd()

    region_names = ""
    for i in boto.glacier.regions():
        region_names = region_names + i.name + " "

    parser = argparse.ArgumentParser(description='Upload a manifest to Amazon Glacier or S3')
    parser.add_argument('--dir', action="store", default=".",
                         help='''Directory containing the manifest. Default: .''')
    parser.add_argument('--aws-secret-access-key', action="store", required=True, help="Your AWS Secret Key")
    parser.add_argument('--aws-access-key-id', action="store", required=True, help="Your AWS Access Key ID")
    parser.add_argument('--vault-name', action="store", required=True, help="Name of the new vault")
    parser.add_argument('--region', action="store", required=True, help="""Glacier region to connect to.
    	                 Available regions: """ + region_names
    	                )
    parser.add_argument('--show-progress',action="store_true",
                        default=False, help='''When this flag is present,
                                               a progress indicator will
                                               show on STDOUT.
                                               Default: False''')
    parser.add_argument('--manifest', action="store", dest="manifest",
    	                default='.manifest.db',
                        help='''Location of the manifast file. Manifests may
                                be located outside the directory indicated by
                                --dir. Must provide path and filename of
                                the manifest file. Default: DIR/.manifest.db''')
    args = parser.parse_args()

    layer2 = boto.glacier.layer2.Layer2(aws_access_key_id=args.aws_access_key_id,
                                        aws_secret_access_key=args.aws_secret_access_key,
                                        region_name=args.region)

    try:
        fur = Furtive(args.dir)
    except:
        raise
    fur.set_manifest(args.manifest)

    try:
        hashes = fur.get_previous_hashes()
    except:
        raise

    if len(hashes) == 0:
        print "No files in manifest. Perhaps you should run hashDir.py to create a manifest first."
        raise Exception("no_manifest")

    vault = layer2.create_vault(args.vault_name)

    file_num = 0
    total_num_files = len(hashes)
    os.chdir(args.dir)
    failed_uploads = []

    for file in hashes.keys():
        if args.show_progress:
            file_num = file_num + 1
            print "\n",file, os.path.getsize(file)
            progress = round((float(file_num) / total_num_files) * 100,1)
            sys.stdout.write("\r" + str(progress) + "% " +
                             str(file_num) + " of " + str(total_num_files))
            sys.stdout.flush()
        attempt = 1
        while attempt < 5:
            try:
                vault.upload_archive(file)
            except UnexpectedHTTPResponseError, e:
                attempt = attempt + 1
                print "Error Uploading %s. Retry %s. Error Deatils %s" % (file,str(attempt),e)
                continue
            print "Tried to upload %s times. Too many errors. Bailing out. (%s)." % (str(attempt),file)
            failed_uploads.append(file)
            break

    #Upload Manifest DB
    vault.upload_archive(fur.manifest_file)

    end_time = time.time()

    if args.show_progress:
        print "\n"

    print "Finished uploading files to the vault %s" % (args.vault_name)
    print "Time taken: %s seconds" % (str(end_time - start_time))
    if len(failed_uploads) > 0:
        print "Failed to upload the following files:"
        for file in failed_uploads:
            print "    %s" % file

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before><commit_msg>Set TimeSeries' default time_series & sample_rate to None. The user absolutely must set these as they are intimately linked. We cannot set them default values just incase the user, on __init__, set's one and not the other. TimeSeries is only used by other classes, never directly so i think this is fair. Lastly, TimeSeries will print an informative error if these 2 params are not available.<commit_after><|endoftext|>


<commit_before>"""
cryptography.py
Author: Daniel Wilson
Credit: Avery, Ethan

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!")
let = []
other = []

def encrypt(P):
    key=input("Key: ")
    P=list(P)
    z=0
    p=[]
    for x in P:
        p.append(associations.index(x))
    #print(p)
    K=[]
    for x in key:
        K.append(associations.index(x))
        z=z+1
    #print(c)
    c=[]
    q=0
    while q<len(p):
        c.append((p[q]+K[q%len(K)])%len(associations))
        q=q+1
    #print(c)
    z=0
    C=[]
    while z<len(p):
        C.append(associations[c[z]])
        z=z+1
    #print(C)
    s=""

    for x in C:
        s=s+x
    print(s)
    return(s)

    for x in C:
        print(x,end="")
    print("")

def decrypt(Z):
    key=input("Key: ")
    Z=list(Z)
    z=0
    p=[]
    for x in Z:
        p.append(associations.index(x))
    #print(p)
    K=[]
    for x in key:
        K.append(associations.index(x))
        z=z+1
    #print(c)
    c=[]
    q=0
    while q<len(p):
        c.append((p[q]-K[q%len(K)])%len(associations))
        q=q+1
    #print(c)
    z=0
    C=[]
    while z<len(p):
        C.append(associations[c[z]])
        z=z+1
    #print(C)
    s=""

    for x in C:
        s=s+x
    print(s)
    return(s)

    for x in C:
        print(x,end="")
    print("")

def request_answer():
    id=input("Enter e to encrypt, d to decrypt, or q to quit")
    if id == "e":
        mes=str(input("Message: "))
        encrypt(mes)
        request_answer()
    elif id == "d":
        mes=str(input("Message: "))
        decrypt(mes)
        request_answer()
    elif id == "q":
        print("Goodbye!")
        return()
    else:
        print("Did not understand command, try again.")
        request_answer()

request_answer()<commit_msg>Updated from Brython Server: 3/8/2016 2:40:25 PM<commit_after>"""
cryptography.py
Author: Daniel Wilson
Credit: Avery, Ethan

Assignment:

Write and submit a program that encrypts and decrypts user data.

See the detailed requirements at https://github.com/HHS-IntroProgramming/Cryptography/blob/master/README.md
"""
associations = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;'\"/\\<>(){}[]-=_+?!")
let = []
other = []

def encrypt(P):
    key=input("Key: ")
    P=list(P)
    z=0
    p=[]
    for x in P:
        p.append(associations.index(x))
    #print(p)
    K=[]
    for x in key:
        K.append(associations.index(x))
        z=z+1
    #print(c)
    c=[]
    q=0
    while q<len(p):
        c.append((p[q]+K[q%len(K)])%len(associations))
        q=q+1
    #print(c)
    z=0
    C=[]
    while z<len(p):
        C.append(associations[c[z]])
        z=z+1
    #print(C)
    s=""

    for x in C:
        s=s+x
    print(s)
    return(s)

    for x in C:
        print(x,end="")
    print("")

def decrypt(Z):
    key=input("Key: ")
    Z=list(Z)
    z=0
    p=[]
    for x in Z:
        p.append(associations.index(x))
    #print(p)
    K=[]
    for x in key:
        K.append(associations.index(x))
        z=z+1
    #print(c)
    c=[]
    q=0
    while q<len(p):
        c.append((p[q]-K[q%len(K)])%len(associations))
        q=q+1
    #print(c)
    z=0
    C=[]
    while z<len(p):
        C.append(associations[c[z]])
        z=z+1
    #print(C)
    s=""

    for x in C:
        s=s+x
    print(s)
    return(s)

    for x in C:
        print(x,end="")
    print("")

def request_answer():
    id=input("Enter e to encrypt, d to decrypt, or q to quit: ")
    if id == "e":
        mes=str(input("Message: "))
        encrypt(mes)
        request_answer()
    elif id == "d":
        mes=str(input("Message: "))
        decrypt(mes)
        request_answer()
    elif id == "q":
        print("Goodbye!")
        return()
    else:
        print("Did not understand command, try again.")
        request_answer()

request_answer()<|endoftext|>