<commit_before>import argparse
from hashlib import md5
import uuid
import glob
from datetime import datetime
import re
import csv
import os
import platform
import pickle
from pyzbar.pyzbar import decode
from PIL import Image
import cv2
import features

# File extensions that are scanned and logged
INPUT_FILE_TYPES = ('.jpg', '.jpeg', '.JPG', '.JPEG')
# File type extensions that are logged when filename matches a scanned input file
ARCHIVE_FILE_TYPES = ('.CR2', '.cr2', '.RAW', '.raw')
# Barcode symbologies accepted, others ignored
ACCEPTED_SYMBOLOGIES = ['CODE39']
#TODO add accepted barcode string patterns
FIELD_DELIMITER = ',' # delimiter used in output CSV

def md5hash(fname):
    # from https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file
    # using this approach to ensure larger files can be read into memory
    #hash_md5 = hashlib.md5()
    hash_md5 = md5()
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()

def closest_histogram(image_histogram=None):
    candidates = {}
    for model_key, model_histogram in models.items():
        diff = features.chi2_distance(image_histogram, model_histogram)
        candidates[model_key] = diff
    # http://stackoverflow.com/questions/3282823/get-key-with-the-least-value-from-a-dictionary
    # http://stackoverflow.com/a/3282904
    min_value = min(candidates.values())
    min_keys = {}
    for k, v in candidates.items():
        if candidates[k] == min_value:
            min_keys[k] = v
    return min_keys

def creation_date(path_to_file):
    # From https://stackoverflow.com/a/39501288
    """
    Try to get the date that a file was created, falling back to when it was
    last modified if that isn't possible.
    See http://stackoverflow.com/a/39501288/1709587 for explanation.
    """
    if platform.system() == 'Windows':
        return os.path.getctime(path_to_file)
    else:
        stat = os.stat(path_to_file)
        try:
            return stat.st_birthtime
        except AttributeError:
            # We're probably on Linux. No easy way to get creation dates here,
            # so we'll settle for when its content was last modified.
            return stat.st_mtime

# Attempts to get actual path of files with correct case
def get_actual_filename(name):
    # From https://stackoverflow.com/a/14742779
    dirs = name.split('\\')
    # disk letter
    test_name = [dirs[0].upper()]
    for d in dirs[1:]:
        test_name += ["%s[%s]" % (d[:-1], d[-1])]
    res = glob.glob('\\'.join(test_name))
    if not res:
        #File not found
        return None
    return res[0]

def get_actual_filename2(name):
    # from https://stackoverflow.com/a/7133137
    name = "%s[%s]" % (name[:-1], name[-1])
    return glob.glob(name)#[0]

def casedpath(path):
    # from https://stackoverflow.com/a/35229734
    r = glob.glob(re.sub(r'([^:/\\])(?=[/\\]|$)', r'[\1]', path))
    return r and r[0] or path

def log_file_data(batch_id=None, batch_path=None, closest_model=None,\
    image_event_id=None, barcodes=None, image_classifications=None, \
    image_path=None, file_uuid=None, derived_from_file=None):
    basename = os.path.basename(image_path)
    file_name, file_extension = os.path.splitext(basename)
    # Get file creation time
    file_creation_time = datetime.fromtimestamp(creation_date(image_path))
    #generate MD5 hash
    file_hash = md5hash(image_path)
    #generate UUID for image
    #file_uuid = uuid.uuid4()

    reportWriter.writerow([\
    batch_id, batch_path, \
    image_event_id, barcodes, image_classifications, closest_model, \
    image_path, basename, file_name, file_extension, file_creation_time, file_hash, file_uuid, derived_from_file])

# set up argument parser
ap = argparse.ArgumentParser()
ap.add_argument("-s", "--source", required=True, \
    help="Path to the directory that contains the images to be analyzed.")
ap.add_argument("-m", "--models", required=False, \
    help="Path to the model file for folder identification through histogram analysis.")
ap.add_argument("-o", "--output", required=False, \
    help="Path to the directory where log file is written.")
args = vars(ap.parse_args())

analysis_start_time = datetime.now()
batch_id = str(uuid.uuid4())
batch_path = os.path.realpath(args["source"])
# load models from pickle file
if args["models"]:
    print('Loading model histograms.')
    models = pickle.load(open(args["models"], "rb"))

# Create file for results
log_file_name = analysis_start_time.date().isoformat() + '_' + batch_id + '.csv'# Convert date to string in ISO format
# Test output path
if args["output"] is not None:
    output_directory = os.path.realpath(args["output"])
    print('output_directory:', output_directory)
    #TODO make sure directory exists and is writeable
    log_file_path = os.path.join(output_directory, log_file_name)
    reportFile = open(log_file_path, "w")
else:
    reportFile = open(log_file_name, "w") # will default to write in location where script is executed

reportWriter = csv.writer(reportFile, delimiter = FIELD_DELIMITER, escapechar='#')
# write header
reportWriter.writerow([\
    "batch_id", "batch_path", \
    "image_event_id", "barcodes", "image_classifications", "closest_model",\
    "image_path", "basename", "file_name", "file_extension", "file_creation_time", "file_hash", "file_uuid", "derived_from_file"])
    #"ImagePath", "DirPath" , "BaseName", "FileName", "FileExtension", "Code", "CodeType" , "Scan time"])

#TODO extract information from directory name (imager, station, etc)
#iterate JPG files in directory passed from args
directory_path = os.path.realpath(args["source"])
files_analyzed = 0
print('Scanning directory:', directory_path)
#TODO change image search to use INPUT_FILE_TYPES
for image_path in glob.glob(directory_path + "/*.JPG"): #this file search seems to be case sensitive
    files_analyzed += 1
    scan_start_time = datetime.now()
    image_event_id = str(uuid.uuid4())
    image_classifications = None
    # Gather file data
    #TODO getting basename and file name is done twice, maybe simplify?
    basename = os.path.basename(image_path)
    file_name, file_extension = os.path.splitext(basename)
    # check if a companion archive file exists
    # iterate through potential extensions for archive files
    arch_file_path = None
    for archive_extension in ARCHIVE_FILE_TYPES:
        potential_arch_file_name = file_name + archive_extension
        potential_arch_file_path = os.path.join(directory_path, potential_arch_file_name)
        # test if archive file exists
        # TODO change filename comparison be case-sensitive

        if os.path.exists(potential_arch_file_path):
            # Trying to get the path with correct case, file extension in particular
            """
            print('Matched ext:', archive_extension, 'Matched filename:', potential_arch_file_name)
            print(os.path.realpath(potential_arch_file_path))
            print(os.path.abspath(potential_arch_file_path))
            print(casedpath(potential_arch_file_path))
            arch_file_path = get_actual_filename2(potential_arch_file_path)
            """
            arch_file_path = potential_arch_file_path
            #TODO generate hash, uuid, read creation time, etc
            break

    # print filepaths
    print('Image file:', os.path.basename(image_path))
    if arch_file_path is None:
        print('Archive file: NOT FOUND')
    else:
        #print('Archive file:', arch_file_path)
        print('Archive file:', os.path.basename(arch_file_path))

    # read barcodes from JPG
    barcodes = decode(Image.open(image_path))
    matching_barcodes = []
    best_match = None
    if barcodes:
        image_classifications = 'barcoded_specimen'
        for barcode in barcodes:
            if str(barcode.type) in ACCEPTED_SYMBOLOGIES:
                symbology_type = str(barcode.type)
                data = barcode.data.decode('UTF-8')
                matching_barcodes.append({'type':symbology_type, 'data':data})
                print(symbology_type, data)
    else:
        print('No barcodes found')
        image_classifications = 'folder'
        # compare to folder models
        # save matching models, ranked
        # generate suggested filename? Or should that be done in rename script?
        if args["models"] is None:
            print('No model file provided. No histogram analysis done.')
        else:
            image = cv2.imread(image_path)
            histogram = features.describe(image)
            best_match = closest_histogram(image_histogram=histogram)
            print ('best_match', best_match)
    # TODO record scan finish time
    scan_end_time = datetime.now()
    #print(scan_end_time)
    # TODO report analysis progress and ETA

    #log CR2 data
    if arch_file_path:
        arch_file_uuid = uuid.uuid4()
        log_file_data(batch_id=batch_id, batch_path=batch_path, closest_model=best_match,\
            image_event_id=image_event_id, barcodes=matching_barcodes, image_classifications=image_classifications, \
            image_path=arch_file_path, file_uuid=arch_file_uuid)
    else:
        arch_file_uuid = None

    #log JPG data
    derivative_file_uuid = uuid.uuid4()
    log_file_data(batch_id=batch_id, batch_path=batch_path, closest_model=best_match,\
        image_event_id=image_event_id, barcodes=matching_barcodes, image_classifications=image_classifications, \
        image_path=image_path, file_uuid=derivative_file_uuid, derived_from_file=arch_file_uuid)

analysis_end_time = datetime.now()
print('Started:', analysis_start_time)
print('Completed:', analysis_end_time)
print('Files analyized:', files_analyzed)
print('Duration:', analysis_end_time - analysis_start_time)
if files_analyzed > 0:
    print('Time per file:', (analysis_end_time - analysis_start_time)/files_analyzed)
print('Report written to:',log_file_name)
<commit_msg>better handling of folder and specimen recording<commit_after>import argparse
from hashlib import md5
import uuid
import glob
from datetime import datetime
import re
import csv
import os
import platform
import pickle
from pyzbar.pyzbar import decode
from PIL import Image
import cv2
import features

# File extensions that are scanned and logged
INPUT_FILE_TYPES = ('.jpg', '.jpeg', '.JPG', '.JPEG')
# File type extensions that are logged when filename matches a scanned input file
ARCHIVE_FILE_TYPES = ('.CR2', '.cr2', '.RAW', '.raw')
# Barcode symbologies accepted, others ignored
ACCEPTED_SYMBOLOGIES = ['CODE39']
#TODO add accepted barcode string patterns
FIELD_DELIMITER = ',' # delimiter used in output CSV

def md5hash(fname):
    # from https://stackoverflow.com/questions/3431825/generating-an-md5-checksum-of-a-file
    # using this approach to ensure larger files can be read into memory
    #hash_md5 = hashlib.md5()
    hash_md5 = md5()
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()

def closest_histogram(image_histogram=None):
    candidates = {}
    for model_key, model_histogram in models.items():
        diff = features.chi2_distance(image_histogram, model_histogram)
        candidates[model_key] = diff
    # http://stackoverflow.com/questions/3282823/get-key-with-the-least-value-from-a-dictionary
    # http://stackoverflow.com/a/3282904
    min_value = min(candidates.values())
    min_keys = {}
    for k, v in candidates.items():
        if candidates[k] == min_value:
            min_keys[k] = v
    return min_keys

def creation_date(path_to_file):
    # From https://stackoverflow.com/a/39501288
    """
    Try to get the date that a file was created, falling back to when it was
    last modified if that isn't possible.
    See http://stackoverflow.com/a/39501288/1709587 for explanation.
    """
    if platform.system() == 'Windows':
        return os.path.getctime(path_to_file)
    else:
        stat = os.stat(path_to_file)
        try:
            return stat.st_birthtime
        except AttributeError:
            # We're probably on Linux. No easy way to get creation dates here,
            # so we'll settle for when its content was last modified.
            return stat.st_mtime

# Attempts to get actual path of files with correct case
def get_actual_filename(name):
    # From https://stackoverflow.com/a/14742779
    dirs = name.split('\\')
    # disk letter
    test_name = [dirs[0].upper()]
    for d in dirs[1:]:
        test_name += ["%s[%s]" % (d[:-1], d[-1])]
    res = glob.glob('\\'.join(test_name))
    if not res:
        #File not found
        return None
    return res[0]

def get_actual_filename2(name):
    # from https://stackoverflow.com/a/7133137
    name = "%s[%s]" % (name[:-1], name[-1])
    return glob.glob(name)#[0]

def casedpath(path):
    # from https://stackoverflow.com/a/35229734
    r = glob.glob(re.sub(r'([^:/\\])(?=[/\\]|$)', r'[\1]', path))
    return r and r[0] or path

def log_file_data(batch_id=None, batch_path=None, closest_model=None,\
    image_event_id=None, barcodes=None, image_classifications=None, \
    image_path=None, file_uuid=None, derived_from_file=None):
    basename = os.path.basename(image_path)
    file_name, file_extension = os.path.splitext(basename)
    # Get file creation time
    file_creation_time = datetime.fromtimestamp(creation_date(image_path))
    #generate MD5 hash
    file_hash = md5hash(image_path)
    #generate UUID for image
    #file_uuid = uuid.uuid4()

    reportWriter.writerow([\
    batch_id, batch_path, \
    image_event_id, barcodes, image_classifications, closest_model, \
    image_path, basename, file_name, file_extension, file_creation_time, file_hash, file_uuid, derived_from_file])

# set up argument parser
ap = argparse.ArgumentParser()
ap.add_argument("-s", "--source", required=True, \
    help="Path to the directory that contains the images to be analyzed.")
ap.add_argument("-m", "--models", required=False, \
    help="Path to the model file for folder identification through histogram analysis.")
ap.add_argument("-o", "--output", required=False, \
    help="Path to the directory where log file is written.")
args = vars(ap.parse_args())

analysis_start_time = datetime.now()
batch_id = str(uuid.uuid4())
batch_path = os.path.realpath(args["source"])
# load models from pickle file
if args["models"]:
    print('Loading model histograms.')
    models = pickle.load(open(args["models"], "rb"))

# Create file for results
log_file_name = analysis_start_time.date().isoformat() + '_' + batch_id + '.csv'# Convert date to string in ISO format
# Test output path
if args["output"] is not None:
    output_directory = os.path.realpath(args["output"])
    print('output_directory:', output_directory)
    #TODO make sure directory exists and is writeable
    log_file_path = os.path.join(output_directory, log_file_name)
    reportFile = open(log_file_path, "w")
else:
    reportFile = open(log_file_name, "w") # will default to write in location where script is executed

reportWriter = csv.writer(reportFile, delimiter = FIELD_DELIMITER, escapechar='#')
# write header
reportWriter.writerow([\
    "batch_id", "batch_path", \
    "image_event_id", "barcodes", "image_classifications", "closest_model",\
    "image_path", "basename", "file_name", "file_extension", "file_creation_time", "file_hash", "file_uuid", "derived_from_file"])
    #"ImagePath", "DirPath" , "BaseName", "FileName", "FileExtension", "Code", "CodeType" , "Scan time"])

#TODO extract information from directory name (imager, station, etc)
#iterate JPG files in directory passed from args
directory_path = os.path.realpath(args["source"])
files_analyzed = 0
print('Scanning directory:', directory_path)
#TODO change image search to use INPUT_FILE_TYPES
for image_path in glob.glob(directory_path + "/*.JPG"): #this file search seems to be case sensitive
    files_analyzed += 1
    scan_start_time = datetime.now()
    image_event_id = str(uuid.uuid4())
    image_classifications = None
    # Gather file data
    #TODO getting basename and file name is done twice, maybe simplify?
    basename = os.path.basename(image_path)
    file_name, file_extension = os.path.splitext(basename)
    # check if a companion archive file exists
    # iterate through potential extensions for archive files
    arch_file_path = None
    for archive_extension in ARCHIVE_FILE_TYPES:
        potential_arch_file_name = file_name + archive_extension
        potential_arch_file_path = os.path.join(directory_path, potential_arch_file_name)
        # test if archive file exists
        # TODO change filename comparison be case-sensitive

        if os.path.exists(potential_arch_file_path):
            # Trying to get the path with correct case, file extension in particular
            """
            print('Matched ext:', archive_extension, 'Matched filename:', potential_arch_file_name)
            print(os.path.realpath(potential_arch_file_path))
            print(os.path.abspath(potential_arch_file_path))
            print(casedpath(potential_arch_file_path))
            arch_file_path = get_actual_filename2(potential_arch_file_path)
            """
            arch_file_path = potential_arch_file_path
            #TODO generate hash, uuid, read creation time, etc
            break

    # print filepaths
    print('Image file:', os.path.basename(image_path))
    if arch_file_path is None:
        print('Archive file: NOT FOUND')
    else:
        #print('Archive file:', arch_file_path)
        print('Archive file:', os.path.basename(arch_file_path))

    # read barcodes from JPG
    barcodes = decode(Image.open(image_path))
    matching_barcodes = []
    image_classifications = []
    best_match = None
    if barcodes:
        image_classifications.append('barcoded')
        for barcode in barcodes:
            if str(barcode.type) in ACCEPTED_SYMBOLOGIES:
                symbology_type = str(barcode.type)
                data = barcode.data.decode('UTF-8')
                matching_barcodes.append({'type':symbology_type, 'data':data})
                print(symbology_type, data)
    else:
        print('No barcodes found')
        image_classifications.append('folder')
        # compare to folder models
        # save matching models, ranked
        # generate suggested filename? Or should that be done in rename script?
        if args["models"] is None:
            print('No model file provided. No histogram analysis done.')
        else:
            image = cv2.imread(image_path)
            histogram = features.describe(image)
            best_match = closest_histogram(image_histogram=histogram)
            [(model_name, model_similarity)] = best_match.items()
            if 'specimen' in model_name:
                image_classifications.append('ambiguous')
            print ('best_match', best_match)
    # TODO record scan finish time
    scan_end_time = datetime.now()
    #print(scan_end_time)
    # TODO report analysis progress and ETA

    #log CR2 data
    image_classifications_string = ",".join(image_classifications)
    if arch_file_path:
        arch_file_uuid = uuid.uuid4()
        log_file_data(batch_id=batch_id, batch_path=batch_path, closest_model=best_match,\
            image_event_id=image_event_id, barcodes=matching_barcodes, image_classifications=image_classifications_string, \
            image_path=arch_file_path, file_uuid=arch_file_uuid)
    else:
        arch_file_uuid = None

    #log JPG data
    derivative_file_uuid = uuid.uuid4()
    log_file_data(batch_id=batch_id, batch_path=batch_path, closest_model=best_match,\
        image_event_id=image_event_id, barcodes=matching_barcodes, image_classifications=image_classifications_string, \
        image_path=image_path, file_uuid=derivative_file_uuid, derived_from_file=arch_file_uuid)

analysis_end_time = datetime.now()
print('Started:', analysis_start_time)
print('Completed:', analysis_end_time)
print('Files analyized:', files_analyzed)
print('Duration:', analysis_end_time - analysis_start_time)
if files_analyzed > 0:
    print('Time per file:', (analysis_end_time - analysis_start_time)/files_analyzed)
print('Report written to:',log_file_name)
<|endoftext|>


<commit_before>import json

class Client(object):
    def __init__(self, access_token):
        self.access_token = access_token

class FacebookObject(object):
    def __init__(self, obj):
        if isinstance(obj, basestring):
            obj = json.loads(obj)
        for key in obj:
            setattr(self, key, self._process_datapoint(obj[key]))

    def _process_datapoint(self, data):
        if isinstance(data, list):
            data = [self._process_datapoint(entry) for entry in data]
        elif isinstance(data, dict):
            if 'id' in data:
                data = FacebookObject(data)
        return data<commit_msg>Automatically fetch data from facebook<commit_after>import json

import requests

class Client(object):
    def __init__(self, access_token):
        self.access_token = access_token

    def get(self, id):
        print 'fetching for %s' % id
        return json.loads(requests.get('https://graph.facebook.com/%s?access_token=%s' %(id, self.access_token)).content)

class FacebookObject(object):
    def __init__(self, obj):
        if isinstance(obj, basestring):
            obj = json.loads(obj)
        for key in obj:
            setattr(self, key, self._process_datapoint(obj[key]))

    def __getattr__(self, name):
        if hasattr(self, 'id'):
            client = Client('AAAFEAjFZCzHUBALPONCwLZA2GBXIkm1joYkB0rZANqbcU83iILOzwexL5DreZAJcCBxRiorhZC6JlUY6fDZANyGxKcIzKzVEtk9G1psiHZB1gZDZD')
            self.__dict__ = FacebookObject(client.get(self.id)).__dict__
            return self.__getattribute__(name)
        raise AttributeError

    def _process_datapoint(self, data):
        if isinstance(data, list):
            data = [self._process_datapoint(entry) for entry in data]
        elif isinstance(data, dict):
            if 'id' in data:
                data = FacebookObject(data)
            else:
                for key in data:
                    data[key] = self._process_datapoint(data[key])
        return data<|endoftext|>


<commit_before>__version__ = '0.2.3'<commit_msg>bump version to v0.2.4<commit_after>__version__ = '0.2.4'<|endoftext|>


<commit_before>#
# Find first occurance of a number in a sorted array (increasing order)
# Approach- Binary Search
# T(n)- O(log n)
#

import unittest

def firstOccurance(array, query):
    lo, hi = 0, len(array) - 1
    while lo <= hi:
        mid = (hi + lo) // 2
        #print("lo: ", lo, " hi: ", hi, " mid: ", mid)
        if lo == hi:
            break
        if array[mid] < query:
            lo = mid + 1
        else:
            hi = mid
    if array[lo] == query:
      return lo

class TestSuite(unittest.TestCase):

    def test_firstOccurance(self):
        def helper(array, query):
            idx = array.index(query) if query in array else None
            return idx
        array = [1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 6, 6, 6]
        self.assertEqual(firstOccurance(array, 1), helper(array, 1))
        self.assertEqual(firstOccurance(array, 3), helper(array, 3))
        self.assertEqual(firstOccurance(array, 5), helper(array, 5))
        self.assertEqual(firstOccurance(array, 6), helper(array, 6))
        self.assertEqual(firstOccurance(array, 7), helper(array, 7))
        self.assertEqual(firstOccurance(array, -1), helper(array, -1))

if __name__ == '__main__':

    unittest.main()
<commit_msg>Update first_occurance.py<commit_after>#
# Find first occurance of a number in a sorted array (increasing order)
# Approach- Binary Search
# T(n)- O(log n)
#

import unittest

def firstOccurance(array, query):
    lo, hi = 0, len(array) - 1
    while lo < hi:
        mid = lo + (hi - lo) // 2
        #print("lo: ", lo, " hi: ", hi, " mid: ", mid)
        if array[mid] < query:
            lo = mid + 1
        else:
            hi = mid
    if lo < len(array) and array[lo] == query:
      return lo

class TestSuite(unittest.TestCase):

    def test_firstOccurance(self):
        def helper(array, query):
            idx = array.index(query) if query in array else None
            return idx
        array = [1, 1, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 6, 6, 6]
        self.assertEqual(firstOccurance(array, 1), helper(array, 1))
        self.assertEqual(firstOccurance(array, 3), helper(array, 3))
        self.assertEqual(firstOccurance(array, 5), helper(array, 5))
        self.assertEqual(firstOccurance(array, 6), helper(array, 6))
        self.assertEqual(firstOccurance(array, 7), helper(array, 7))
        self.assertEqual(firstOccurance(array, -1), helper(array, -1))

if __name__ == '__main__':

    unittest.main()
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import unicode_literals
from django.apps import apps
from django.contrib.auth.models import AbstractUser, UserManager
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.validators import MaxLengthValidator
from django.db.models import (
    BooleanField, permalink, TextField, URLField,
    CharField, ForeignKey, PositiveIntegerField, ImageField)
from django.db.models.signals import class_prepared
from django.dispatch import receiver
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ungettext_lazy, ugettext_lazy as _
from mptt.fields import TreeForeignKey
from mptt.models import MPTTModel
from libretto.models.base import AutoriteModel, CommonTreeManager, \
    CommonTreeQuerySet
from common.utils.html import href, sc
from common.utils.text import str_list_w_last

#
# Hack to monkey patch the user model
#

LAST_NAME_MAX_LENGTH = 75

@receiver(class_prepared)
def longer_last_name(sender, *args, **kwargs):
    if sender.__name__ == 'HierarchicUser' \
            and sender.__module__ == 'accounts.models':
        last_name = sender._meta.get_field('last_name')
        last_name.max_length = LAST_NAME_MAX_LENGTH
        assert len(last_name.validators) == 1
        last_name.validators = [MaxLengthValidator(LAST_NAME_MAX_LENGTH)]

#
# Regular modelisation
#

class HierarchicUserQuerySet(CommonTreeQuerySet):
    def html(self, tags=True):
        return str_list_w_last([u.html(tags=tags) for u in self])

class HierarchicUserManager(CommonTreeManager, UserManager):
    queryset_class = HierarchicUserQuerySet

    def html(self, tags=True):
        return self.get_queryset().html(tags=tags)

def _get_valid_modelnames_func(autorites_only=True):
    def is_valid(model):
        b = hasattr(model, 'link')
        if autorites_only:
            return b and AutoriteModel in model.__bases__
        return b

    def get_valid_modelnames():
        models = frozenset(apps.get_models())
        return [model.__name__.lower() for model in models if is_valid(model)]
    return get_valid_modelnames

@python_2_unicode_compatible
class HierarchicUser(MPTTModel, AbstractUser):
    show_email = BooleanField(_('afficher l’email'), default=False)
    website = URLField(_('site internet'), blank=True)
    website_verbose = CharField(
        _('nom affiché du site internet'), max_length=50, blank=True)

    legal_person = BooleanField(
        _('personne morale'), default=False,
        help_text=_('Cochez si vous êtes une institution ou un ensemble.'))
    content_type = ForeignKey(
        ContentType, blank=True, null=True,
        limit_choices_to={'model__in': _get_valid_modelnames_func()},
        verbose_name=_('type d’autorité associée'))
    object_id = PositiveIntegerField(_('identifiant de l’autorité associée'),
                                     blank=True, null=True)
    content_object = GenericForeignKey()

    mentor = TreeForeignKey(
        'self', null=True, blank=True, related_name='disciples',
        verbose_name=_('mentor'),
        limit_choices_to={'willing_to_be_mentor__exact': True})
    willing_to_be_mentor = BooleanField(
        _('Veut être mentor'), default=False)

    avatar = ImageField(_('avatar'), upload_to='avatars/',
                        blank=True, null=True)

    presentation = TextField(
        _('présentation'), blank=True, validators=[MaxLengthValidator(5000)])
    fonctions = TextField(_('fonctions au sein de l’équipe'), blank=True,
                          validators=[MaxLengthValidator(200)])
    literature = TextField(_('publications'), blank=True)

    objects = HierarchicUserManager()

    class MPTTMeta(object):
        parent_attr = 'mentor'
        order_insertion_by = ('last_name', 'first_name', 'username')

    class Meta(object):
        ordering = ('last_name', 'first_name')
        verbose_name = ungettext_lazy('utilisateur', 'utilisateurs', 1)
        verbose_name_plural = ungettext_lazy('utilisateur', 'utilisateurs', 2)

    def __str__(self, tags=False):
        return self.html(tags=False)

    def get_full_name(self, tags=False):
        full_name = '%s %s' % (self.first_name, sc(self.last_name, tags=tags))
        return full_name.strip()

    def html(self, tags=True):
        txt = self.get_full_name(tags=tags) or self.get_username()
        return href(self.get_absolute_url(), txt, tags=tags)

    def link(self, tags=True):
        return self.html(tags=tags)

    @permalink
    def get_absolute_url(self):
        return 'user_profile', (self.username,)

    def website_link(self):
        return href(self.website, self.website_verbose or self.website,
                    new_tab=True)

    def email_link(self):
        return href('mailto:' + self.email, self.email)

    def dossiersdevenements_roots(self):
        return self.dossierdevenements.exclude(parent__owner=self)

    @staticmethod
    def autocomplete_search_fields():
        return ('first_name__unaccent__icontains',
                'last_name__unaccent__icontains')
<commit_msg>Workaround for django-mptt/django-mptt#382.<commit_after># coding: utf-8

from __future__ import unicode_literals
from django.apps import apps
from django.contrib.auth.models import AbstractUser, UserManager
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from django.core.validators import MaxLengthValidator
from django.db.models import (
    BooleanField, permalink, TextField, URLField,
    CharField, ForeignKey, PositiveIntegerField, ImageField)
from django.db.models.signals import class_prepared
from django.dispatch import receiver
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ungettext_lazy, ugettext_lazy as _
from mptt.fields import TreeForeignKey
from mptt.models import MPTTModel
from mptt.utils import _get_tree_model

from common.utils.html import href, sc
from common.utils.text import str_list_w_last
from libretto.models.base import (
    AutoriteModel, CommonTreeManager, CommonTreeQuerySet, CommonManager,
)

#
# Hack to monkey patch the user model
#

LAST_NAME_MAX_LENGTH = 75

@receiver(class_prepared)
def longer_last_name(sender, *args, **kwargs):
    if sender.__name__ == 'HierarchicUser' \
            and sender.__module__ == 'accounts.models':
        last_name = sender._meta.get_field('last_name')
        last_name.max_length = LAST_NAME_MAX_LENGTH
        assert len(last_name.validators) == 1
        last_name.validators = [MaxLengthValidator(LAST_NAME_MAX_LENGTH)]

#
# Regular modelisation
#

class HierarchicUserQuerySet(CommonTreeQuerySet):
    def html(self, tags=True):
        return str_list_w_last([u.html(tags=tags) for u in self])

class HierarchicUserManager(CommonTreeManager, UserManager):
    queryset_class = HierarchicUserQuerySet
    use_in_migrations = False

    # FIXME: This is a workaround to https://github.com/django-mptt/django-mptt/issues/382
    #        Remove when fixed upstream.
    def contribute_to_class(self, model, name):
        CommonManager.contribute_to_class(self, model, name)

        if not model._meta.abstract:
            self.tree_model = _get_tree_model(model)

            self._base_manager = None
            if self.tree_model is not None and self.tree_model is not model:
                self._base_manager = self.tree_model._tree_manager

    def html(self, tags=True):
        return self.get_queryset().html(tags=tags)

def _get_valid_modelnames_func(autorites_only=True):
    def is_valid(model):
        b = hasattr(model, 'link')
        if autorites_only:
            return b and AutoriteModel in model.__bases__
        return b

    def get_valid_modelnames():
        models = frozenset(apps.get_models())
        return [model.__name__.lower() for model in models if is_valid(model)]
    return get_valid_modelnames

@python_2_unicode_compatible
class HierarchicUser(MPTTModel, AbstractUser):
    show_email = BooleanField(_('afficher l’email'), default=False)
    website = URLField(_('site internet'), blank=True)
    website_verbose = CharField(
        _('nom affiché du site internet'), max_length=50, blank=True)

    legal_person = BooleanField(
        _('personne morale'), default=False,
        help_text=_('Cochez si vous êtes une institution ou un ensemble.'))
    content_type = ForeignKey(
        ContentType, blank=True, null=True,
        limit_choices_to={'model__in': _get_valid_modelnames_func()},
        verbose_name=_('type d’autorité associée'))
    object_id = PositiveIntegerField(_('identifiant de l’autorité associée'),
                                     blank=True, null=True)
    content_object = GenericForeignKey()

    mentor = TreeForeignKey(
        'self', null=True, blank=True, related_name='disciples',
        verbose_name=_('mentor'),
        limit_choices_to={'willing_to_be_mentor__exact': True})
    willing_to_be_mentor = BooleanField(
        _('Veut être mentor'), default=False)

    avatar = ImageField(_('avatar'), upload_to='avatars/',
                        blank=True, null=True)

    presentation = TextField(
        _('présentation'), blank=True, validators=[MaxLengthValidator(5000)])
    fonctions = TextField(_('fonctions au sein de l’équipe'), blank=True,
                          validators=[MaxLengthValidator(200)])
    literature = TextField(_('publications'), blank=True)

    objects = HierarchicUserManager()

    class MPTTMeta(object):
        parent_attr = 'mentor'
        order_insertion_by = ('last_name', 'first_name', 'username')

    class Meta(object):
        ordering = ('last_name', 'first_name')
        verbose_name = ungettext_lazy('utilisateur', 'utilisateurs', 1)
        verbose_name_plural = ungettext_lazy('utilisateur', 'utilisateurs', 2)

    def __str__(self, tags=False):
        return self.html(tags=False)

    def get_full_name(self, tags=False):
        full_name = '%s %s' % (self.first_name, sc(self.last_name, tags=tags))
        return full_name.strip()

    def html(self, tags=True):
        txt = self.get_full_name(tags=tags) or self.get_username()
        return href(self.get_absolute_url(), txt, tags=tags)

    def link(self, tags=True):
        return self.html(tags=tags)

    @permalink
    def get_absolute_url(self):
        return 'user_profile', (self.username,)

    def website_link(self):
        return href(self.website, self.website_verbose or self.website,
                    new_tab=True)

    def email_link(self):
        return href('mailto:' + self.email, self.email)

    def dossiersdevenements_roots(self):
        return self.dossierdevenements.exclude(parent__owner=self)

    @staticmethod
    def autocomplete_search_fields():
        return ('first_name__unaccent__icontains',
                'last_name__unaccent__icontains')
<|endoftext|>


<commit_before>from __future__ import division,print_function
import sys,random,re
sys.dont_write_bytecode =True

def rows(file):
  """Leaps over any columns marked 'skip'.
  Turn strings to numbers or strings.
  Kill comments. Join lines that end in 'sep'."""
  skip, sep  = '?', ','
  bad = r'(["\' \t\r\n]|#.*)'
  def atom(x):
    try : return int(x)
    except ValueError:
       try : return float(x)
       except ValueError : return x
  def worker():
    n,kept = 0,""
    for line in open(file):
      now   = re.sub(bad,"",line)
      kept += now
      if kept:
        if not now[-1] == sep:
          yield n, map(atom, kept.split(sep))
          n += 1
          kept = ""
  todo = None
  for n,line in worker():
    todo = todo or [col for col,name in enumerate(line)
                    if not skip in name]
    yield n, [ line[col] for col in todo ]

class o:
  def __init__(i,**d): i.update(**d)
  def update(i,**d): i.__dict__.update(**d); return i
  def __repr__(i)   :
    d    = i.__dict__
    show = [':%s %s' % (k,d[k])
            for k in sorted(d.keys() )
            if k[0] is not "_"]
    return '{'+' '.join(show)+'}'

def genic0(**d): return o(
    k=10,
    era=100,
    num='$',
    klass='=',
    seed=1).update(**d)

def header(w,row):
  for col,val in enumerate(row):
    ako = w.num if w.opt.num in val else w.sym
    ako.append(col)
    what= w.dep if w.opt.klass in val else w.indep
    what.append(col)
    w.name[col] = val
    w.index[val] = col

def data(w,row):
  for col in w.num:
    val = row[col]
    o.min[col] = min(val, o.min.get(col,10**32))
    o.max[col] = max(val, o.max.get(col,-10**32))

def nearest(w,row):
  def norm(val,col):
    lo, hi = w.min[col], w.max[old]
    return (val - lo ) / (hi - lo + 0.00001)
  def dist(a,b,w):
    n,d = 0,0
    for col in w.num:
      x1,x2 = a[col], b[col]
      n1,n2 = norm(x1,col), norm(x2,col)
      d    += (n1 - n2)**2
      n    += 1
    for col in w.sym:
      x1,x2 = a[col],b[col]
      d    += (0 if x1 == x2 else 1)
      n    += 1
    return d**0.5 / n**0.5
  lo, out = 10**32, None
  for n,(_,centroid) in enumerate(w.centroids):
    d = dist(row,centroid,w)
    if d < lo:
      lo,out = d,n
  return out

def move(w,row1,n):
  w2,row2 = w.centroids[n]
  w1 = 1

def genic(datafile='data/diabetes.csv',opt=None):
  w = o(num=[], sym=[], dep=[], indep=[],
        centroids=[],
        min={}, max={}, name={},index={},
        opt=None or genic0())
  for n,row in rows(datafile):
    if n == 0:
      header(w,row)
    else:
      data(w,row)
      if len(o.centroids) < o.opt.k:
        w.centroids += [(1,row)]
      else:
        move(w,row,nearest(w,row))

genic()

<commit_msg>genic in pthon<commit_after>from __future__ import division,print_function
import sys,random,re
sys.dont_write_bytecode =True

def genic0(**d): return o(
    k=20,
    era=100,
    num='$',
    klass='=',
    seed=1).update(**d)

rand= random.random
seed= random.seed

def say(c):
  sys.stdout.write(c); sys.stdout.flush()

def rows(file):
  """Leaps over any columns marked 'skip'.
  Turn strings to numbers or strings.
  Kill comments. Join lines that end in 'sep'."""
  skip, sep  = '?', ','
  bad = r'(["\' \t\r\n]|#.*)'
  def atom(x):
    try : return int(x)
    except ValueError:
       try : return float(x)
       except ValueError : return x
  def worker():
    n,kept = 0,""
    for line in open(file):
      now   = re.sub(bad,"",line)
      kept += now
      if kept:
        if not now[-1] == sep:
          yield n, map(atom, kept.split(sep))
          n += 1
          kept = ""
  todo = None
  for n,line in worker():
    todo = todo or [col for col,name in enumerate(line)
                    if not skip in name]
    yield n, [ line[col] for col in todo ]

class o:
  """Standard trick for defining a bag of names slots
  that have no methods."""
  def __init__(i,**d): i.update(**d)
  def update(i,**d): i.__dict__.update(**d); return i
  def __repr__(i)   :
    d    = i.__dict__
    show = [':%s %s' % (k,d[k])
            for k in sorted(d.keys() )
            if k[0] is not "_"]
    return '{'+' '.join(show)+'}'

def header(w,row):
  def numOrSym(val):
    return w.num if w.opt.num in val else w.sym
  def indepOrDep(val):
    return w.dep if w.opt.klass in val else w.indep
  for col,val in enumerate(row):
    numOrSym(val).append(col)
    indepOrDep(val).append(col)
    w.name[col] = val
    w.index[val] = col

def data(w,row):
  for col in w.num:
    val = row[col]
    w.min[col] = min(val, w.min.get(col,val))
    w.max[col] = max(val, w.max.get(col,val))

def nearest(w,row):
  def norm(val,col):
    lo, hi = w.min[col], w.max[col]
    return (val - lo ) / (hi - lo + 0.00001)
  def dist(a,b,w):
    n,d = 0,0
    for col in w.num:
      x1,x2 = a[col], b[col]
      n1,n2 = norm(x1,col), norm(x2,col)
      d    += (n1 - n2)**2
      n    += 1
    for col in w.sym:
      x1,x2 = a[col],b[col]
      d    += (0 if x1 == x2 else 1)
      n    += 1
    return d**0.5 / n**0.5
  lo, out = 10**32, None
  for n,(_,centroid) in enumerate(w.centroids):
    d = dist(row,centroid,w)
    if d < lo:
      lo,out = d,n
  return out

def move(w,row1,n):
  u0,row0 = w.centroids[n]
  u1 = 1
  out = [None]*len(row1)
  for col in w.sym:
    x0,x1 = row0[col], row1[col]
    out[col] = x1 if rand() < 1/(u0+u1) else x0
  for col in w.num:
    x0,x1= row0[col], row1[col]
    out[col] = (u0*x0 + u1*x1)/ (u0+u1)
  w.centroids[n] = (u0 + u1, out)

def less(w) :
  b4 = len(w.centroids)
  all = normu(w)
  w.centroids = [(1,row) for u,row in all if u >= rand()]
  now=len(w.centroids)
  print(" - ",b4 - now)

def normu(w):
  all  = sorted([(u,row) for u,row in w.centroids])
  most = all[-1][0]
  return [(u/most,row) for u,row in all]

def genic(datafile='data/diabetes.csv',opt=None):
  w = o(num=[], sym=[], dep=[], indep=[],
        centroids=[],
        min={}, max={}, name={},index={},
        opt=None or genic0())
  for n,row in rows(datafile):
    if n == 0:
      header(w,row)
    else:
      data(w,row)
      if len(w.centroids) < w.opt.k:
        say("+")
        w.centroids += [(1,row)]
      else:
        move(w,row,nearest(w,row))
      if 0 == (n % w.opt.era):
        less(w)
  return sorted(normu(w),reverse=True)

def g3(row):
  for col,val in enumerate(row):
    if isinstance(val,float):
      val = round(n,3)
    row[col] = val
  return row

for n,row in genic():
  print(n,":",g3(row))

<|endoftext|>


<commit_before># Copyright (c) 2017 Weitian LI <liweitianux@live.com>
# MIT license
#
# Weitian LI
# 2017-02-11

"""
Manage the observation manifest in YAML format.

NOTE
----
Use `ruamel.yaml`_ instead of `PyYAML`_ to preserve the comments
and other structures in the YAML file.

.. _`ruamel.yaml`: https://bitbucket.org/ruamel/yaml
.. _`PyYAML`: http://pyyaml.org/
"""

import os
from collections import OrderedDict
import argparse

import ruamel.yaml

class Manifest:
    """
    Manage the observational products manifest.
    """
    def __init__(self, filepath):
        self.filepath = os.path.abspath(filepath)
        self.manifest = ruamel.yaml.load(
            open(filepath), Loader=ruamel.yaml.RoundTripLoader)
        if self.manifest is None:
            self.manifest = ruamel.yaml.comments.CommentedMap()

    def dump(self):
        return ruamel.yaml.dump(self.manifest,
                                Dumper=ruamel.yaml.RoundTripDumper)

    def save(self):
        with open(self.filepath, "w") as f:
            f.write(self.dump())

    def show(self):
        print(self.dump())

    def has(self, key):
        return True if key in self.manifest else False

    def get(self, key):
        """
        Get the value of the specified item in the manifest.

        Parameters
        ----------
        key : str
            The key of the item to be requested.

        Raises
        ------
        KeyError :
            If the specified item doesn't exist.
        """
        if self.has(key):
            return self.manifest[key]
        else:
            raise KeyError("manifest doesn't have item: '%s'" % key)

    def gets(self, keys, default=None, splitlist=False):
        """
        Get the value of the specified item in the manifest.

        Parameters
        ----------
        keys : list[str]
            A list of keys specifying the items to be requested.
        default : optional
            The default value to return if the item not exists.
        splitlist : bool, optional
            Split the item value if it is a list, making it is easier
            to export as CSV format.

        Returns
        -------
        data : `~OrderedDict`
            Ordered dictionary containing the requested items.

        Returns
        -------
        """
        data = OrderedDict([
            (key, self.manifest.get(key, default)) for key in keys
        ])
        if splitlist:
            ds = OrderedDict()
            for k, v in data.items():
                if isinstance(v, list):
                    for i, vi in enumerate(v):
                        ki = "{0}[{1}]".format(k, i)
                        ds[ki] = vi
                else:
                    ds[k] = v
            data = ds
        return data

    def getpath(self, key, relative=False, sep=None):
        """
        Get the absolute path to the specified item by joining
        with the location of this manifest file.

        Parameters
        ----------
        key : str
            Key of the file item
        relative : bool, optional
            Whether return the file path(s) as relative path w.r.t.
            the current working directory?
        sep : str, optional
            If not ``None``, join the multiple file paths with the
            specified separator, instead of returning a list of paths.
        """
        value = self.get(key)
        cwd = os.getcwd()
        if isinstance(value, list):
            path = [os.path.join(os.path.dirname(self.filepath), f)
                    for f in value]
            if relative:
                path = [os.path.relpath(p, start=cwd) for p in path]
            if sep is not None:
                path = sep.join(path)
        else:
            path = os.path.join(os.path.dirname(self.filepath), value)
            if relative:
                path = os.path.relpath(path, start=cwd)
        return path

    def set(self, key, value):
        """
        Set the value of the specified item in the manifest.
        (Will add a new item or update an existing item.)
        """
        self.manifest[key] = self.parse_value(value)
        self.save()

    def setpath(self, key, path):
        """
        Get the relative path of the given file w.r.t. this manifest file,
        and set it to be the value of the specified item.
        (Will add a new item or update an existing item.)
        """
        dirname = os.path.dirname(self.filepath)
        if isinstance(path, list):
            abspath = [os.path.abspath(p) for p in path]
            relpath = [os.path.relpath(p, start=dirname) for p in abspath]
            if len(relpath) == 1:
                relpath = relpath[0]
        else:
            abspath = os.path.abspath(path)
            relpath = os.path.relpath(abspath, start=dirname)
        self.manifest[key] = relpath
        self.save()

    def add(self, key, value):
        """
        Add the specified new item in the manifest.

        If the specified item already exists, raise a ``KeyError``.
        """
        if key in self.manifest:
            raise KeyError("manifest already has item: '%s'" % key)
        else:
            self.set(key, value)

    def update(self, key, value):
        """
        Update the specified existing item in the manifest.

        If the specified item doesn't exist, raise a ``KeyError``.
        """
        if key in self.manifest:
            self.set(key, value)
        else:
            raise KeyError("manifest doesn't have item: '%s'" % key)

    def delete(self, key):
        """
        Delete the specified item from the manifest.
        """
        del self.manifest[key]
        self.save()

    @classmethod
    def parse_value(self, value):
        """
        Parse the given (list of) value(s) from string.

        If the value list only has length of 1, then just return
        the only element without list enclosure.
        """
        if isinstance(value, list):
            pv = [self.parse_value_single(v) for v in value]
            if len(pv) == 1:
                return pv[0]
            else:
                return pv
        else:
            return self.parse_value_single(value)

    @staticmethod
    def parse_value_single(value):
        """
        Try to parse the given value from string to integer, float, boolean
        or string.

        If the input value is not string, then just ignore the parse.
        """
        if not isinstance(value, str):
            return value

        try:
            v = int(value)
        except ValueError:
            try:
                v = float(value)
            except ValueError:
                # string/boolean
                if value.lower() in ["true", "yes"]:
                    v = True
                elif value.lower() in ["false", "no"]:
                    v = False
                else:
                    v = value  # string
        return v

def find_manifest(filename="manifest.yaml", startdir=os.getcwd()):
    """
    Find the specified manifest file in current directory and
    the upper-level directories.

    Parameters
    ----------
    filename : str, optional
        Filename of the manifest file (default: ``manifest.yaml``)

    Returns
    -------
    filepath : str
        Absolute path to the manifest file if found.

    Raises
    ------
    FileNotFoundError :
        Cannot found the specified manifest
    """
    dirname = startdir
    filepath = os.path.join(dirname, filename)
    while dirname != "/":
        if os.path.exists(filepath):
            return filepath
        # go upper by one level
        dirname = os.path.dirname(dirname)
        filepath = os.path.join(dirname, filename)
    # not found
    raise FileNotFoundError("cannot found manifest file: %s" % filename)

def get_manifest(filename="manifest.yaml"):
    """
    Find the manifest file and return the Manifest instance of it.

    Parameters
    ----------
    filename : str, optional
        Filename of the manifest file (default: ``manifest.yaml``)

    Returns
    -------
    manifest : `~Manifest`
        Manifest instance of the found manifest file.
    """
    return Manifest(find_manifest(filename))

# COMMAND-LINE INTERFACE ----------------------------------------------------

def cmd_show(args, manifest):
    """
    Default sub-command "show": Show manifest contents.
    """
    manifest.show()

def cmd_get(args, manifest):
    """
    Sub-command "get": Get the value of an item in the manifest.
    """
    if not args.brief:
        print("%s:" % args.key, end=" ")
    value = manifest.get(args.key)
    if isinstance(value, list):
        if args.field:
            print(value[args.field-1])
        else:
            vs = [str(v) for v in value]
            print(args.separator.join(vs))
    else:
        print(value)

def cmd_getpath(args, manifest):
    """
    Sub-command "getpath": Get the absolute path to the specified file
    item in the manifest.
    """
    if not args.brief:
        print("%s:" % args.key, end=" ")
    path = manifest.getpath(args.key, relative=args.relative,
                            sep=args.separator)
    print(path)

def cmd_set(args, manifest):
    """
    Sub-command "set": Set the value of an item in the manifest.
    (Will add a new item or update an existing item.)
    """
    manifest.set(args.key, args.value)
    if not args.brief:
        print("Set item '{0}': {1}".format(args.key, manifest.get(args.key)))

def cmd_setpath(args, manifest):
    """
    Sub-command "setpath": Set the specified file item in the manifest
    to be the relative path of the given file w.r.t. the manifest file.
    """
    manifest.setpath(args.key, args.value)
    if not args.brief:
        print("Set file item '{0}': {1}".format(args.key,
                                                manifest.get(args.key)))

def cmd_add(args, manifest):
    """
    Sub-command "add": Add a new item to the manifest.
    """
    manifest.add(args.key, args.value)
    if not args.brief:
        print("Added item '{0}': {1}".format(args.key, manifest.get(args.key)))

def cmd_update(args, manifest):
    """
    Sub-command "update": Update the value of an existing item in the
    manifest.
    """
    value_old = manifest.get(args.key)
    manifest.update(args.key, args.value)
    if not args.brief:
        print("Updated item '{0}': {1} -> {2}".format(
              args.key, value_old, manifest.get(args.key)))

def cmd_delete(args, manifest):
    """
    Sub-command "delete": Delete an item from the manifest.
    """
    manifest.delete(args.key)
    if not args.brief:
        print("Deleted item: %s" % args.key)

def main(description="Manage the observation manifest (YAML format)",
         default_file="manifest.yaml"):
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument("-F", "--file", dest="file", default=default_file,
                        help="Manifest file (default: %s)" % default_file)
    parser.add_argument("-b", "--brief", dest="brief",
                        action="store_true", help="Be brief")
    parser.add_argument("-C", "--directory", dest="directory", default=".",
                        help="From where to find the manifest file")
    parser.add_argument("-s", "--separator", dest="separator", default=" ",
                        help="Separator to join output list values " +
                        "(default: whitespace)")
    subparsers = parser.add_subparsers(dest="cmd_name",
                                       title="sub-commands",
                                       help="additional help")
    # sub-command: show
    parser_show = subparsers.add_parser("show", help="Show manifest contents")
    parser_show.set_defaults(func=cmd_show)
    # sub-command: get
    parser_get = subparsers.add_parser("get", help="Get an item from manifest")
    parser_get.add_argument("-f", "--field", dest="field", type=int,
                            help="which field to get (default: all fields)")
    parser_get.add_argument("key", help="key of the item")
    parser_get.set_defaults(func=cmd_get)
    # sub-command: getpath
    parser_getpath = subparsers.add_parser(
        "getpath", help="Get the path to a file item from manifest")
    parser_getpath.add_argument("-r", "--relative", dest="relative",
                                action="store_true",
                                help="Return relative path w.r.t. current " +
                                "working directory instead of absolute path")
    parser_getpath.add_argument("key", help="key of the file item")
    parser_getpath.set_defaults(func=cmd_getpath)
    # sub-command: set
    parser_set = subparsers.add_parser(
        "set", help="Set (add/update) an item in manifest")
    parser_set.add_argument("key", help="key of the item")
    parser_set.add_argument("value", nargs="+",
                            help="value of the item")
    parser_set.set_defaults(func=cmd_set)
    # sub-command: setpath
    parser_setpath = subparsers.add_parser(
        "setpath", help="Set a file item using relative path of given files")
    parser_setpath.add_argument("key", help="key of the file item")
    parser_setpath.add_argument("value", nargs="+",
                                help="paths to the files")
    parser_setpath.set_defaults(func=cmd_setpath)
    # sub-command: add
    parser_add = subparsers.add_parser(
        "add", help="Add a new item to manifest")
    parser_add.add_argument("key", help="key of the item")
    parser_add.add_argument("value", nargs="+",
                            help="value of the item")
    parser_add.set_defaults(func=cmd_add)
    # sub-command: update
    parser_update = subparsers.add_parser(
        "update", help="Update an existing item in manifest")
    parser_update.add_argument("key", help="key of the item")
    parser_update.add_argument("value", nargs="+",
                               help="new value of the item")
    parser_update.set_defaults(func=cmd_update)
    # sub-command: delete
    parser_delete = subparsers.add_parser(
        "delete", help="Delete item from manifest")
    parser_delete.add_argument("key", help="key of the item")
    parser_delete.set_defaults(func=cmd_delete)
    #
    args = parser.parse_args()

    if os.path.exists(args.file):
        manifest_file = args.file
    else:
        manifest_file = find_manifest(
            args.file, startdir=os.path.abspath(args.directory))

    manifest = Manifest(manifest_file)

    if args.cmd_name:
        # Dispatch sub-commands to call its specified function
        args.func(args, manifest)
    else:
        cmd_show(None, manifest)
<commit_msg>acispy/manifest.py: Support rename an item key<commit_after># Copyright (c) 2017 Weitian LI <liweitianux@live.com>
# MIT license
#
# Weitian LI
# 2017-02-11

"""
Manage the observation manifest in YAML format.

NOTE
----
Use `ruamel.yaml`_ instead of `PyYAML`_ to preserve the comments
and other structures in the YAML file.

.. _`ruamel.yaml`: https://bitbucket.org/ruamel/yaml
.. _`PyYAML`: http://pyyaml.org/
"""

import os
from collections import OrderedDict
import argparse

import ruamel.yaml

class Manifest:
    """
    Manage the observational products manifest.
    """
    def __init__(self, filepath):
        self.filepath = os.path.abspath(filepath)
        self.manifest = ruamel.yaml.load(
            open(filepath), Loader=ruamel.yaml.RoundTripLoader)
        if self.manifest is None:
            self.manifest = ruamel.yaml.comments.CommentedMap()

    def dump(self):
        return ruamel.yaml.dump(self.manifest,
                                Dumper=ruamel.yaml.RoundTripDumper)

    def save(self):
        with open(self.filepath, "w") as f:
            f.write(self.dump())

    def show(self):
        print(self.dump())

    def has(self, key):
        return True if key in self.manifest else False

    def get(self, key):
        """
        Get the value of the specified item in the manifest.

        Parameters
        ----------
        key : str
            The key of the item to be requested.

        Raises
        ------
        KeyError :
            If the specified item doesn't exist.
        """
        if self.has(key):
            return self.manifest[key]
        else:
            raise KeyError("manifest doesn't have item: '%s'" % key)

    def gets(self, keys, default=None, splitlist=False):
        """
        Get the value of the specified item in the manifest.

        Parameters
        ----------
        keys : list[str]
            A list of keys specifying the items to be requested.
        default : optional
            The default value to return if the item not exists.
        splitlist : bool, optional
            Split the item value if it is a list, making it is easier
            to export as CSV format.

        Returns
        -------
        data : `~OrderedDict`
            Ordered dictionary containing the requested items.

        Returns
        -------
        """
        data = OrderedDict([
            (key, self.manifest.get(key, default)) for key in keys
        ])
        if splitlist:
            ds = OrderedDict()
            for k, v in data.items():
                if isinstance(v, list):
                    for i, vi in enumerate(v):
                        ki = "{0}[{1}]".format(k, i)
                        ds[ki] = vi
                else:
                    ds[k] = v
            data = ds
        return data

    def getpath(self, key, relative=False, sep=None):
        """
        Get the absolute path to the specified item by joining
        with the location of this manifest file.

        Parameters
        ----------
        key : str
            Key of the file item
        relative : bool, optional
            Whether return the file path(s) as relative path w.r.t.
            the current working directory?
        sep : str, optional
            If not ``None``, join the multiple file paths with the
            specified separator, instead of returning a list of paths.
        """
        value = self.get(key)
        cwd = os.getcwd()
        if isinstance(value, list):
            path = [os.path.join(os.path.dirname(self.filepath), f)
                    for f in value]
            if relative:
                path = [os.path.relpath(p, start=cwd) for p in path]
            if sep is not None:
                path = sep.join(path)
        else:
            path = os.path.join(os.path.dirname(self.filepath), value)
            if relative:
                path = os.path.relpath(path, start=cwd)
        return path

    def set(self, key, value):
        """
        Set the value of the specified item in the manifest.
        (Will add a new item or update an existing item.)
        """
        self.manifest[key] = self.parse_value(value)
        self.save()

    def setpath(self, key, path):
        """
        Get the relative path of the given file w.r.t. this manifest file,
        and set it to be the value of the specified item.
        (Will add a new item or update an existing item.)
        """
        dirname = os.path.dirname(self.filepath)
        if isinstance(path, list):
            abspath = [os.path.abspath(p) for p in path]
            relpath = [os.path.relpath(p, start=dirname) for p in abspath]
            if len(relpath) == 1:
                relpath = relpath[0]
        else:
            abspath = os.path.abspath(path)
            relpath = os.path.relpath(abspath, start=dirname)
        self.manifest[key] = relpath
        self.save()

    def add(self, key, value):
        """
        Add the specified new item in the manifest.

        If the specified item already exists, raise a ``KeyError``.
        """
        if key in self.manifest:
            raise KeyError("manifest already has item: '%s'" % key)
        else:
            self.set(key, value)

    def update(self, key, value):
        """
        Update the specified existing item in the manifest.

        If the specified item doesn't exist, raise a ``KeyError``.
        """
        if key in self.manifest:
            self.set(key, value)
        else:
            raise KeyError("manifest doesn't have item: '%s'" % key)

    def delete(self, key):
        """
        Delete the specified item from the manifest.
        """
        del self.manifest[key]
        self.save()

    def rename(self, oldkey, newkey):
        """
        Rename the specified key in the manifest.

        If the specified old key doesn't exist, or the new key already
        exists, the error ``KeyError`` is raised.
        """
        if oldkey not in self.manifest:
            raise KeyError("manifest doesn't have item: '%s'" % oldkey)
        if newkey in self.manifest:
            raise KeyError("manifest already has item: '%s'" % newkey)
        # Add item with the new key
        self.manifest[newkey] = self.manifest[oldkey]
        # Delete old item
        del self.manifest[oldkey]
        self.save()

    @classmethod
    def parse_value(self, value):
        """
        Parse the given (list of) value(s) from string.

        If the value list only has length of 1, then just return
        the only element without list enclosure.
        """
        if isinstance(value, list):
            pv = [self.parse_value_single(v) for v in value]
            if len(pv) == 1:
                return pv[0]
            else:
                return pv
        else:
            return self.parse_value_single(value)

    @staticmethod
    def parse_value_single(value):
        """
        Try to parse the given value from string to integer, float, boolean
        or string.

        If the input value is not string, then just ignore the parse.
        """
        if not isinstance(value, str):
            return value

        try:
            v = int(value)
        except ValueError:
            try:
                v = float(value)
            except ValueError:
                # string/boolean
                if value.lower() in ["true", "yes"]:
                    v = True
                elif value.lower() in ["false", "no"]:
                    v = False
                else:
                    v = value  # string
        return v

def find_manifest(filename="manifest.yaml", startdir=os.getcwd()):
    """
    Find the specified manifest file in current directory and
    the upper-level directories.

    Parameters
    ----------
    filename : str, optional
        Filename of the manifest file (default: ``manifest.yaml``)

    Returns
    -------
    filepath : str
        Absolute path to the manifest file if found.

    Raises
    ------
    FileNotFoundError :
        Cannot found the specified manifest
    """
    dirname = startdir
    filepath = os.path.join(dirname, filename)
    while dirname != "/":
        if os.path.exists(filepath):
            return filepath
        # go upper by one level
        dirname = os.path.dirname(dirname)
        filepath = os.path.join(dirname, filename)
    # not found
    raise FileNotFoundError("cannot found manifest file: %s" % filename)

def get_manifest(filename="manifest.yaml"):
    """
    Find the manifest file and return the Manifest instance of it.

    Parameters
    ----------
    filename : str, optional
        Filename of the manifest file (default: ``manifest.yaml``)

    Returns
    -------
    manifest : `~Manifest`
        Manifest instance of the found manifest file.
    """
    return Manifest(find_manifest(filename))

# COMMAND-LINE INTERFACE ----------------------------------------------------

def cmd_show(args, manifest):
    """
    Default sub-command "show": Show manifest contents.
    """
    manifest.show()

def cmd_get(args, manifest):
    """
    Sub-command "get": Get the value of an item in the manifest.
    """
    if not args.brief:
        print("%s:" % args.key, end=" ")
    value = manifest.get(args.key)
    if isinstance(value, list):
        if args.field:
            print(value[args.field-1])
        else:
            vs = [str(v) for v in value]
            print(args.separator.join(vs))
    else:
        print(value)

def cmd_getpath(args, manifest):
    """
    Sub-command "getpath": Get the absolute path to the specified file
    item in the manifest.
    """
    if not args.brief:
        print("%s:" % args.key, end=" ")
    path = manifest.getpath(args.key, relative=args.relative,
                            sep=args.separator)
    print(path)

def cmd_set(args, manifest):
    """
    Sub-command "set": Set the value of an item in the manifest.
    (Will add a new item or update an existing item.)
    """
    manifest.set(args.key, args.value)
    if not args.brief:
        print("Set item '{0}': {1}".format(args.key, manifest.get(args.key)))

def cmd_setpath(args, manifest):
    """
    Sub-command "setpath": Set the specified file item in the manifest
    to be the relative path of the given file w.r.t. the manifest file.
    """
    manifest.setpath(args.key, args.value)
    if not args.brief:
        print("Set file item '{0}': {1}".format(args.key,
                                                manifest.get(args.key)))

def cmd_add(args, manifest):
    """
    Sub-command "add": Add a new item to the manifest.
    """
    manifest.add(args.key, args.value)
    if not args.brief:
        print("Added item '{0}': {1}".format(args.key, manifest.get(args.key)))

def cmd_update(args, manifest):
    """
    Sub-command "update": Update the value of an existing item in the
    manifest.
    """
    value_old = manifest.get(args.key)
    manifest.update(args.key, args.value)
    if not args.brief:
        print("Updated item '{0}': {1} -> {2}".format(
              args.key, value_old, manifest.get(args.key)))

def cmd_delete(args, manifest):
    """
    Sub-command "delete": Delete an item from the manifest.
    """
    manifest.delete(args.key)
    if not args.brief:
        print("Deleted item: %s" % args.key)

def cmd_rename(args, manifest):
    """
    Sub-command "rename": Rename an item in the manifest.
    """
    manifest.rename(args.key, args.newkey)
    if not args.brief:
        print("Renamed item key: '%s' -> '%s'" % (args.key, args.newkey))

def main(description="Manage the observation manifest (YAML format)",
         default_file="manifest.yaml"):
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument("-F", "--file", dest="file", default=default_file,
                        help="Manifest file (default: %s)" % default_file)
    parser.add_argument("-b", "--brief", dest="brief",
                        action="store_true", help="Be brief")
    parser.add_argument("-C", "--directory", dest="directory", default=".",
                        help="From where to find the manifest file")
    parser.add_argument("-s", "--separator", dest="separator", default=" ",
                        help="Separator to join output list values " +
                        "(default: whitespace)")
    subparsers = parser.add_subparsers(dest="cmd_name",
                                       title="sub-commands",
                                       help="additional help")
    # sub-command: show
    parser_show = subparsers.add_parser("show", help="Show manifest contents")
    parser_show.set_defaults(func=cmd_show)
    # sub-command: get
    parser_get = subparsers.add_parser("get", help="Get an item from manifest")
    parser_get.add_argument("-f", "--field", dest="field", type=int,
                            help="which field to get (default: all fields)")
    parser_get.add_argument("key", help="key of the item")
    parser_get.set_defaults(func=cmd_get)
    # sub-command: getpath
    parser_getpath = subparsers.add_parser(
        "getpath", help="Get the path to a file item from manifest")
    parser_getpath.add_argument("-r", "--relative", dest="relative",
                                action="store_true",
                                help="Return relative path w.r.t. current " +
                                "working directory instead of absolute path")
    parser_getpath.add_argument("key", help="key of the file item")
    parser_getpath.set_defaults(func=cmd_getpath)
    # sub-command: set
    parser_set = subparsers.add_parser(
        "set", help="Set (add/update) an item in manifest")
    parser_set.add_argument("key", help="key of the item")
    parser_set.add_argument("value", nargs="+",
                            help="value of the item")
    parser_set.set_defaults(func=cmd_set)
    # sub-command: setpath
    parser_setpath = subparsers.add_parser(
        "setpath", help="Set a file item using relative path of given files")
    parser_setpath.add_argument("key", help="key of the file item")
    parser_setpath.add_argument("value", nargs="+",
                                help="paths to the files")
    parser_setpath.set_defaults(func=cmd_setpath)
    # sub-command: add
    parser_add = subparsers.add_parser(
        "add", help="Add a new item to manifest")
    parser_add.add_argument("key", help="key of the item")
    parser_add.add_argument("value", nargs="+",
                            help="value of the item")
    parser_add.set_defaults(func=cmd_add)
    # sub-command: update
    parser_update = subparsers.add_parser(
        "update", help="Update an existing item in manifest")
    parser_update.add_argument("key", help="key of the item")
    parser_update.add_argument("value", nargs="+",
                               help="new value of the item")
    parser_update.set_defaults(func=cmd_update)
    # sub-command: delete
    parser_delete = subparsers.add_parser(
        "delete", help="Delete an item from the manifest")
    parser_delete.add_argument("key", help="key of the item")
    parser_delete.set_defaults(func=cmd_delete)
    # sub-command: rename
    parser_rename = subparsers.add_parser(
        "rename", help="Rename an item in the manifest")
    parser_rename.add_argument("key", help="old item key")
    parser_rename.add_argument("newkey", help="new item key")
    parser_rename.set_defaults(func=cmd_rename)
    #
    args = parser.parse_args()

    if os.path.exists(args.file):
        manifest_file = args.file
    else:
        manifest_file = find_manifest(
            args.file, startdir=os.path.abspath(args.directory))

    manifest = Manifest(manifest_file)

    if args.cmd_name:
        # Dispatch sub-commands to call its specified function
        args.func(args, manifest)
    else:
        cmd_show(None, manifest)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

bind='127.0.0.1:3030'
workers=1
timeout=60
capture_output=True

def pre_request(worker, req):
    if req.path.startswith('/content/'):
        req.query = 'path=' + req.path[9:]
        req.path = '/content'
    if req.path.startswith('/taskContent/'):
        parts = req.path.split("/")
        req.query = 'path=' + '/'.join(parts[4:])
        req.path = '/task-content/' + parts[2] + '/' + (parts[3] if len(parts) > 3 else '')

<commit_msg>4 workers.<commit_after># -*- coding: utf-8 -*-

bind='127.0.0.1:3030'
workers=4
timeout=60
capture_output=True

def pre_request(worker, req):
    if req.path.startswith('/content/'):
        req.query = 'path=' + req.path[9:]
        req.path = '/content'
    if req.path.startswith('/taskContent/'):
        parts = req.path.split("/")
        req.query = 'path=' + '/'.join(parts[4:])
        req.path = '/task-content/' + parts[2] + '/' + (parts[3] if len(parts) > 3 else '')

<|endoftext|>


<commit_before>import sys
import argparse

from sqf.parser import parse
import sqf.analyser
from sqf.exceptions import SQFParserError

def analyze(code, writer=sys.stdout):
    try:
        result = parse(code)
    except SQFParserError as e:
        writer.write('[%d,%d]:%s\n' % (e.position[0], e.position[1] - 1, e.message))
        return

    exceptions = sqf.analyser.analyze(result).exceptions
    for e in exceptions:
        writer.write('[%d,%d]:%s\n' % (e.position[0], e.position[1] - 1, e.message))

def _main():
    parser = argparse.ArgumentParser(description="Static Analyser of SQF code")
    parser.add_argument('filename', nargs='?', type=argparse.FileType('r'), default=None,
                        help='The full path of the file to be analyzed')

    args = parser.parse_args()

    if args.filename is not None:
        with open(args.filename) as file:
            code = file.read()
    else:
        code = sys.stdin.read()

    analyze(code)

if __name__ == "__main__":
    _main()
<commit_msg>Fix parsing file - FileType already read<commit_after>import sys
import argparse

from sqf.parser import parse
import sqf.analyser
from sqf.exceptions import SQFParserError

def analyze(code, writer=sys.stdout):
    try:
        result = parse(code)
    except SQFParserError as e:
        writer.write('[%d,%d]:%s\n' % (e.position[0], e.position[1] - 1, e.message))
        return

    exceptions = sqf.analyser.analyze(result).exceptions
    for e in exceptions:
        writer.write('[%d,%d]:%s\n' % (e.position[0], e.position[1] - 1, e.message))

def _main():
    parser = argparse.ArgumentParser(description="Static Analyser of SQF code")
    parser.add_argument('file', nargs='?', type=argparse.FileType('r'), default=None,
                        help='The full path of the file to be analyzed')

    args = parser.parse_args()

    if args.file is not None:
        code = args.file.read()
    else:
        code = sys.stdin.read()

    analyze(code)

if __name__ == "__main__":
    _main()
<|endoftext|>