<commit_before># -*- coding: utf-8 -*-
from collections import OrderedDict

ORGANISMS = [
    {
    'uuid': '7745b647-ff15-4ff3-9ced-b897d4e2983c',
    'name': 'human',
    'scientific_name': 'Homo sapiens',
    'taxon_id': "9606",
    },
    {
    'uuid': '3413218c-3d86-498b-a0a2-9a406638e786',
    'name': 'mouse',
    'scientific_name': 'Mus musculus',
    'taxon_id': "10090",
    },
]

TARGETS = [
    {
        'uuid': 'dcd60c9f-7f2e-4d75-8276-9c9a9c6c7669',
        'label': 'ATF4',
        'organism': '7745b647-ff15-4ff3-9ced-b897d4e2983c',  # looked up on insert?
        'gene_name': 'ATF4',
        # 'aliases': [
        #     {'alias': 'CREB2', 'source': 'HGNC'},
        #     {'alias': 'TXREB', 'source': 'HGNC'},
        #     {'alias': 'CREB-2', 'source': 'HGNC'},
        #     {'alias': 'TAXREB67', 'source': 'HGNC'},
        # ],
        # 'dbxref': {
        #     'UniProtKB': ['Q96AQ3'],
        #     'HGNC': ['786']
        # }
    },
    {
        'uuid': 'BAF56297-9628-418F-B78E-95EDD524E4F6',
        'label': 'H3K4me3',
        'organism': '7745b647-ff15-4ff3-9ced-b897d4e2983c',  # looked up on insert?
        'gene_name': 'H3F3A',
        # 'aliases': [
        #     {'alias': 'H3.3', 'source': 'HGNC'},
        #     {'alias': 'H3F3', 'source': 'HGNC'},
        #     {'alias': 'histone H3.3', 'source': 'HGNC'},
        # ],
        # 'dbxref': {
        #     'UniProtKB': ['P84243'],
        #     'HGNC': ['4764']
        # },
   },
]

SOURCES = [
    {
    'uuid': '3aa827c3-92f8-41fa-9608-201558f7a1c4',
    'name': 'sigma',
    'title': 'Sigma-Aldrich',
    'url': 'http://www.sigmaaldrich.com',
    },
    {
    'uuid': '1d5be796-8f80-4fd4-b6c7-6674318657eb',
    'name': 'gingeras',
    'title': 'Gingeras Lab',
    'url': 'http://www.gingeraslab.edu',
    },
]

ANTIBODY_LOTS = [
    {
    'uuid': 'bc293400-eab3-41fb-a41e-35552686b67d',
    'clonality': 'monoclonal',
    'host_organism': 'mouse',
    'source': 'sigma',  # PK
    'product_id': 'WH0000468M1',  # PK
    'lot_id': 'CB191-2B3',  # PK
    'url': 'http://www.sigmaaldrich.com/catalog/product/sigma/wh0000468m1?lang=en&region=US',
    'isotype': u'IgG1κ',
    'antigen_description': 'ATF4 (NP_001666, a.a. 171-271) partial recombinant protein with GST tag.',
    'lab': 'myers',
    'award': 'Myers',
    },
]

ANTIBODY_CHARACTERIZATIONS = [
    {
    'uuid': 'c4da2e0c-149f-4aee-ac21-8690dfdadb1f',
    'characterizes': 'bc293400-eab3-41fb-a41e-35552686b67d',
    'target': 'dcd60c9f-7f2e-4d75-8276-9c9a9c6c7669',
    'lab': 'myers',
    'award': 'Myers',
    },
]

ANTIBODY_APPROVALS = [
    {
    'uuid': 'a8f94078-2d3b-4647-91a2-8ec91b096708',
    'antibody': 'bc293400-eab3-41fb-a41e-35552686b67d',
    'target': 'dcd60c9f-7f2e-4d75-8276-9c9a9c6c7669',
    'characterizations': [
        'c4da2e0c-149f-4aee-ac21-8690dfdadb1f',
        ],
    'lab': 'myers',
    'award': 'Myers',
    'status': 'SUBMITTED',
    },
]

BIOSAMPLES = [
    {
    'uuid': '7c245cea-7d59-45fb-9ebe-f0454c5fe950',
    'accession': 'ENCBS000TST',
    'biosample_term_id': 'UBERON:349829',
    'biosample_term_name': 'Liver',
    'biosample_type': 'tissue',
    'source': 'gingeras',
    'lab': 'myers',
    'award': 'Myers',
    'organism': 'human',
    },
]

LIBRARIES = [
    {
    'uuid': 'a95b00ea-fc13-42ee-a322-e36793f1325d',
    'accession': 'ENCLB000TST',
    'nucleic_acid_term_id': 'SO:0000352',
    'nucleic_acid_term_name': 'DNA',
    'biosample': 'ENCBS000TST',
    'lab': 'myers',
    'award': 'Myers',
    },
]

AWARDS = [
    {
    'uuid': '529e3e74-3caa-4842-ae64-18c8720e610e',
    'name': 'ENCODE3-DCC',
    },
    {
    'uuid': 'fae1bd8b-0d90-4ada-b51f-0ecc413e904d',
    'name': 'Myers',
    },
]

BAD_AWARDS = [  # UUID same as one of labs
    {
    'uuid': '529e3e74-3caa-4842-ae64-18c8720e610e',
    'name': 'ENCODE3-DCC',
    },
    {
    'uuid': 'b635b4ed-dba3-4672-ace9-11d76a8d03af',
    'name': 'Myers',
    },
]

LABS = [
    {
    'uuid': 'cfb789b8-46f3-4d59-a2b3-adc39e7df93a',
    'name': 'cherry',
    'title': 'Cherry Lab',
    'institute_name': 'Stanford University'
    },
    {
    'uuid': 'b635b4ed-dba3-4672-ace9-11d76a8d03af',
    'name': 'myers',
    'title': 'Myers Lab',
    'institute_name': 'HudsonAlpha Institute for Biotechnology'
    },
]

BAD_LABS = [  # same UUID
    {
    'uuid': '2c334112-288e-4d45-9154-3f404c726daf',
    'name': 'cherry',
    'title': 'Cherry Lab',
    'institute_name': 'Stanford University'
    },
    {
    'uuid': '2c334112-288e-4d45-9154-3f404c726daf',
    'name': 'myers',
    'title': 'Myers Lab',
    'institute_name': 'HudsonAlpha Institute for Biotechnology'
    },
]

USERS = [
    {
    'uuid': 'e9be360e-d1c7-4cae-9b3a-caf588e8bb6f',
    'first_name': 'Benjamin',
    'last_name': 'Hitz',
    'email': 'hitz@stanford.edu',
    'submits_for': ['cherry'],
    'groups': ['admin', 'programmer'],
    },
    {
    'uuid': '1e945b04-aa54-4732-8b81-b41d4565f5f9',
    'first_name': 'Cricket',
    'last_name': 'Sloan',
    'email': 'cricket@stanford.edu',
    'submits_for': ['cfb789b8-46f3-4d59-a2b3-adc39e7df93a'],
    'groups': ['admin', 'wrangler'],
    },
    {
    'uuid': 'bb319896-3f78-4e24-b6e1-e4961822bc9b',
    'first_name': 'Florencia',
    'last_name': 'Pauli-Behn',
    'email': 'paulibehn@hudsonalpha.org',
    'submits_for': ['b635b4ed-dba3-4672-ace9-11d76a8d03af'],
    },
    {
    'uuid': '2aaf75d9-4273-4bd8-9fd1-217e3d0af7cc',
    'first_name': 'No',
    'last_name': 'Login',
    'email': 'nologin@example.org',
    'submits_for': ['b635b4ed-dba3-4672-ace9-11d76a8d03af'],
    'status': 'DISABLED',
    },
]

EXPERIMENTS = [
    {
    'uuid': 'fa3e286d-a44c-41e0-bcce-3c24fe9aa02d',
    'accession': 'ENCSR000TST',
    'lab': 'myers',
    'award': 'Myers',
    },
    {
    'uuid': 'f26eeb63-e77c-47e9-b8fd-e21b24065424',
    'accession': 'ENCSR001TST',
    'lab': 'myers',
    'award': 'Myers',
    },
]

REPLICATES = [
    {
    'uuid': '13468ab5-2369-4857-a303-7c5f28918190',
    'experiment': 'ENCSR000TST',
    'library': 'ENCLB000TST',
    'biological_replicate_number': 1,
    'technical_replicate_number': 1,
    },
    {
    'uuid': 'df38fd69-8c4b-4870-86a4-55f4b355324e',
    'experiment': 'ENCSR000TST',
    'library': 'ENCLB000TST',
    'biological_replicate_number': 1,
    'technical_replicate_number': 2,
    },
]

FILES = [
    {
    'uuid': 'c22e0390-af36-483c-950d-5a2e0efe37ec',
    'accession': 'ENCFF000TST',
    'dataset': 'ENCSR000TST',
    'file_format': 'fastq',
    'md5sum': 'd41d8cd98f00b204e9800998ecf8427e',
    'output_type': 'rawData',
    },
    {
    'uuid': '0a38fea0-c067-415a-9710-c887aff95767',
    'accession': 'ENCFF001TST',
    'dataset': 'ENCSR001TST',
    'file_format': 'fastq',
    'md5sum': 'd41d8cd98f00b204e9800998ecf8427e',
    'output_type': 'rawData',
    },
]

URL_COLLECTION = OrderedDict([
    ('lab', LABS),
    ('user', USERS),
    ('award', AWARDS),
    ('organism', ORGANISMS),
    ('target', TARGETS),
    ('source', SOURCES),
    ('antibody_lot', ANTIBODY_LOTS),
    ('antibody_characterization', ANTIBODY_CHARACTERIZATIONS),
    ('antibody_approval', ANTIBODY_APPROVALS),
    ('biosample', BIOSAMPLES),
    ('library', LIBRARIES),
    ('experiment', EXPERIMENTS),
    ('replicate', REPLICATES),
    ('file', FILES),
])

def load(testapp, item_type):
    items = []
    for item in URL_COLLECTION[item_type]:
        res = testapp.post_json('/' + item_type, item, status=201)
        items.append(res.json['@graph'][0])
    return items
<commit_msg>Updated status in sample data to reflect changes.<commit_after># -*- coding: utf-8 -*-
from collections import OrderedDict

ORGANISMS = [
    {
    'uuid': '7745b647-ff15-4ff3-9ced-b897d4e2983c',
    'name': 'human',
    'scientific_name': 'Homo sapiens',
    'taxon_id': "9606",
    },
    {
    'uuid': '3413218c-3d86-498b-a0a2-9a406638e786',
    'name': 'mouse',
    'scientific_name': 'Mus musculus',
    'taxon_id': "10090",
    },
]

TARGETS = [
    {
        'uuid': 'dcd60c9f-7f2e-4d75-8276-9c9a9c6c7669',
        'label': 'ATF4',
        'organism': '7745b647-ff15-4ff3-9ced-b897d4e2983c',  # looked up on insert?
        'gene_name': 'ATF4',
        # 'aliases': [
        #     {'alias': 'CREB2', 'source': 'HGNC'},
        #     {'alias': 'TXREB', 'source': 'HGNC'},
        #     {'alias': 'CREB-2', 'source': 'HGNC'},
        #     {'alias': 'TAXREB67', 'source': 'HGNC'},
        # ],
        # 'dbxref': {
        #     'UniProtKB': ['Q96AQ3'],
        #     'HGNC': ['786']
        # }
    },
    {
        'uuid': 'BAF56297-9628-418F-B78E-95EDD524E4F6',
        'label': 'H3K4me3',
        'organism': '7745b647-ff15-4ff3-9ced-b897d4e2983c',  # looked up on insert?
        'gene_name': 'H3F3A',
        # 'aliases': [
        #     {'alias': 'H3.3', 'source': 'HGNC'},
        #     {'alias': 'H3F3', 'source': 'HGNC'},
        #     {'alias': 'histone H3.3', 'source': 'HGNC'},
        # ],
        # 'dbxref': {
        #     'UniProtKB': ['P84243'],
        #     'HGNC': ['4764']
        # },
   },
]

SOURCES = [
    {
    'uuid': '3aa827c3-92f8-41fa-9608-201558f7a1c4',
    'name': 'sigma',
    'title': 'Sigma-Aldrich',
    'url': 'http://www.sigmaaldrich.com',
    },
    {
    'uuid': '1d5be796-8f80-4fd4-b6c7-6674318657eb',
    'name': 'gingeras',
    'title': 'Gingeras Lab',
    'url': 'http://www.gingeraslab.edu',
    },
]

ANTIBODY_LOTS = [
    {
    'uuid': 'bc293400-eab3-41fb-a41e-35552686b67d',
    'clonality': 'monoclonal',
    'host_organism': 'mouse',
    'source': 'sigma',  # PK
    'product_id': 'WH0000468M1',  # PK
    'lot_id': 'CB191-2B3',  # PK
    'url': 'http://www.sigmaaldrich.com/catalog/product/sigma/wh0000468m1?lang=en&region=US',
    'isotype': u'IgG1κ',
    'antigen_description': 'ATF4 (NP_001666, a.a. 171-271) partial recombinant protein with GST tag.',
    'lab': 'myers',
    'award': 'Myers',
    },
]

ANTIBODY_CHARACTERIZATIONS = [
    {
    'uuid': 'c4da2e0c-149f-4aee-ac21-8690dfdadb1f',
    'characterizes': 'bc293400-eab3-41fb-a41e-35552686b67d',
    'target': 'dcd60c9f-7f2e-4d75-8276-9c9a9c6c7669',
    'lab': 'myers',
    'award': 'Myers',
    },
]

ANTIBODY_APPROVALS = [
    {
    'uuid': 'a8f94078-2d3b-4647-91a2-8ec91b096708',
    'antibody': 'bc293400-eab3-41fb-a41e-35552686b67d',
    'target': 'dcd60c9f-7f2e-4d75-8276-9c9a9c6c7669',
    'characterizations': [
        'c4da2e0c-149f-4aee-ac21-8690dfdadb1f',
        ],
    'lab': 'myers',
    'award': 'Myers',
    'status': 'PENDING DCC REVIEW',
    },
]

BIOSAMPLES = [
    {
    'uuid': '7c245cea-7d59-45fb-9ebe-f0454c5fe950',
    'accession': 'ENCBS000TST',
    'biosample_term_id': 'UBERON:349829',
    'biosample_term_name': 'Liver',
    'biosample_type': 'tissue',
    'source': 'gingeras',
    'lab': 'myers',
    'award': 'Myers',
    'organism': 'human',
    },
]

LIBRARIES = [
    {
    'uuid': 'a95b00ea-fc13-42ee-a322-e36793f1325d',
    'accession': 'ENCLB000TST',
    'nucleic_acid_term_id': 'SO:0000352',
    'nucleic_acid_term_name': 'DNA',
    'biosample': 'ENCBS000TST',
    'lab': 'myers',
    'award': 'Myers',
    },
]

AWARDS = [
    {
    'uuid': '529e3e74-3caa-4842-ae64-18c8720e610e',
    'name': 'ENCODE3-DCC',
    },
    {
    'uuid': 'fae1bd8b-0d90-4ada-b51f-0ecc413e904d',
    'name': 'Myers',
    },
]

BAD_AWARDS = [  # UUID same as one of labs
    {
    'uuid': '529e3e74-3caa-4842-ae64-18c8720e610e',
    'name': 'ENCODE3-DCC',
    },
    {
    'uuid': 'b635b4ed-dba3-4672-ace9-11d76a8d03af',
    'name': 'Myers',
    },
]

LABS = [
    {
    'uuid': 'cfb789b8-46f3-4d59-a2b3-adc39e7df93a',
    'name': 'cherry',
    'title': 'Cherry Lab',
    'institute_name': 'Stanford University'
    },
    {
    'uuid': 'b635b4ed-dba3-4672-ace9-11d76a8d03af',
    'name': 'myers',
    'title': 'Myers Lab',
    'institute_name': 'HudsonAlpha Institute for Biotechnology'
    },
]

BAD_LABS = [  # same UUID
    {
    'uuid': '2c334112-288e-4d45-9154-3f404c726daf',
    'name': 'cherry',
    'title': 'Cherry Lab',
    'institute_name': 'Stanford University'
    },
    {
    'uuid': '2c334112-288e-4d45-9154-3f404c726daf',
    'name': 'myers',
    'title': 'Myers Lab',
    'institute_name': 'HudsonAlpha Institute for Biotechnology'
    },
]

USERS = [
    {
    'uuid': 'e9be360e-d1c7-4cae-9b3a-caf588e8bb6f',
    'first_name': 'Benjamin',
    'last_name': 'Hitz',
    'email': 'hitz@stanford.edu',
    'submits_for': ['cherry'],
    'groups': ['admin', 'programmer'],
    },
    {
    'uuid': '1e945b04-aa54-4732-8b81-b41d4565f5f9',
    'first_name': 'Cricket',
    'last_name': 'Sloan',
    'email': 'cricket@stanford.edu',
    'submits_for': ['cfb789b8-46f3-4d59-a2b3-adc39e7df93a'],
    'groups': ['admin', 'wrangler'],
    },
    {
    'uuid': 'bb319896-3f78-4e24-b6e1-e4961822bc9b',
    'first_name': 'Florencia',
    'last_name': 'Pauli-Behn',
    'email': 'paulibehn@hudsonalpha.org',
    'submits_for': ['b635b4ed-dba3-4672-ace9-11d76a8d03af'],
    },
    {
    'uuid': '2aaf75d9-4273-4bd8-9fd1-217e3d0af7cc',
    'first_name': 'No',
    'last_name': 'Login',
    'email': 'nologin@example.org',
    'submits_for': ['b635b4ed-dba3-4672-ace9-11d76a8d03af'],
    'status': 'DISABLED',
    },
]

EXPERIMENTS = [
    {
    'uuid': 'fa3e286d-a44c-41e0-bcce-3c24fe9aa02d',
    'accession': 'ENCSR000TST',
    'lab': 'myers',
    'award': 'Myers',
    },
    {
    'uuid': 'f26eeb63-e77c-47e9-b8fd-e21b24065424',
    'accession': 'ENCSR001TST',
    'lab': 'myers',
    'award': 'Myers',
    },
]

REPLICATES = [
    {
    'uuid': '13468ab5-2369-4857-a303-7c5f28918190',
    'experiment': 'ENCSR000TST',
    'library': 'ENCLB000TST',
    'biological_replicate_number': 1,
    'technical_replicate_number': 1,
    },
    {
    'uuid': 'df38fd69-8c4b-4870-86a4-55f4b355324e',
    'experiment': 'ENCSR000TST',
    'library': 'ENCLB000TST',
    'biological_replicate_number': 1,
    'technical_replicate_number': 2,
    },
]

FILES = [
    {
    'uuid': 'c22e0390-af36-483c-950d-5a2e0efe37ec',
    'accession': 'ENCFF000TST',
    'dataset': 'ENCSR000TST',
    'file_format': 'fastq',
    'md5sum': 'd41d8cd98f00b204e9800998ecf8427e',
    'output_type': 'rawData',
    },
    {
    'uuid': '0a38fea0-c067-415a-9710-c887aff95767',
    'accession': 'ENCFF001TST',
    'dataset': 'ENCSR001TST',
    'file_format': 'fastq',
    'md5sum': 'd41d8cd98f00b204e9800998ecf8427e',
    'output_type': 'rawData',
    },
]

URL_COLLECTION = OrderedDict([
    ('lab', LABS),
    ('user', USERS),
    ('award', AWARDS),
    ('organism', ORGANISMS),
    ('target', TARGETS),
    ('source', SOURCES),
    ('antibody_lot', ANTIBODY_LOTS),
    ('antibody_characterization', ANTIBODY_CHARACTERIZATIONS),
    ('antibody_approval', ANTIBODY_APPROVALS),
    ('biosample', BIOSAMPLES),
    ('library', LIBRARIES),
    ('experiment', EXPERIMENTS),
    ('replicate', REPLICATES),
    ('file', FILES),
])

def load(testapp, item_type):
    items = []
    for item in URL_COLLECTION[item_type]:
        res = testapp.post_json('/' + item_type, item, status=201)
        items.append(res.json['@graph'][0])
    return items
<|endoftext|>


<commit_before>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(verbose, url, auth):
    if verbose > 0:
        print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("BeginTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on beginTransaction: {0}".format(response['response']['error']))

    if verbose > 0:
        print('Transaction started.')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(verbose, url, auth, cancel, transactionId):
    if verbose > 0:
        print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

    if verbose > 0:
        print('Transaction ended.')

def checkout(url, auth, branch, transactionId):
    print "Checking out "+branch+" branch..."
    params = {'output_format': 'JSON', 'branch': branch, 'transactionId':transactionId}
    request = make_request(url=url+'checkout.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Checkout for branch "+branch+" failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    #print response
    if response['response']['success']:
        newBranch = response['response']['NewTarget']
        print "Checked out "+newBranch+' branch.'
        return newBranch
    else:
        print "----"
        print "Checkout for "+branch+" branch failed."
        print "Error Message: "+response['response']['error']
        return None

def pollTask(url, auth, taskID):
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    #print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    status = response['task']['status']

    progress = None
    try:
        progress = response['task']['progress']
    except:
        progress = None

    result = None
    try:
        result = response['task']['result']
    except:
        result = None

    errorMessage = None
    try:
        errorMessage = response['task']['error']['message']
    except:
        errorMessage = None

    return status, progress, result, errorMessage

def printTaskStatus(taskID, status, progress, result, errorMessage):

    if status == "RUNNING":
        print "----"
        print "++Task "+str(taskID)+" is running."
        if progress:
            progress_task = progress['task']
            if progress_task == 'Committing features...':
                print progress_task
            elif progress_task == 'Importing into GeoGig repo...':
                print progress_task+' '+str(progress['amount'])+' entities imported'
            else:
                print "Current Step: "+str(progress['task'])
                print "Entities Processed: "+str(progress['amount'])
        if result:
            print "Entities Processed: "+str(result['OSMReport']['processedEntities'])
    elif status == "FAILED":
        print "----"
        print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
    elif status == "FINISHED":
        print "----"
        print "++Task "+str(taskID)+" is finished."
        if result:
            print "Entities Processed: "+str(result['OSMReport']['processedEntities'])
    elif status == "CANCELLED":
        print "----"
        print response
        print "++Task "+str(taskID)+" was cancelled."
        print "Entities Processed: "+str(result['OSMReport']['processedEntities'])
    else:
        print "----"
        print "++Task "+str(taskID)+" is "+status+"."

def cancelTask(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json?cancel=true', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Could not cancel task: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "CANCELLED":
            print "++Task "+str(taskID)+" was cancelled."
        else:
            print "++Error.  Could not cancel task "+str(taskID)+".  Task is "+str(taskStatus)+"."

    return taskStatus;

def waitOnTask(verbose, url, auth, taskID, timeout):
    maxTime = timeout
    timeSlept = 0
    sleepCycle = 5
    taskStatus = None

    print "----------------------------------"
    print "Waiting for task "+str(taskID)+"..."
    print "Maximum wait time is "+str(maxTime)+" seconds."
    while timeSlept < maxTime:
        taskStatus, taskProgress, taskResult, errorMessage = pollTask(url, auth, taskID)
        if verbose > 0:
            printTaskStatus(taskID, taskStatus, taskProgress, taskResult, errorMessage)
        if not (taskStatus in ['WAITING','RUNNING']):
            break
        if verbose > 0:
            print "Time: "+str(timeSlept)+"/"+str(maxTime)
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    if taskStatus in ['WAITING','RUNNING']:
        print "Task "+str(taskID)+" timed out after "+str(timeSlept)+" seconds."
        print "Attempting to cancel task "+str(taskID)
        maxTime = 60
        timeSlept = 0
        sleepCycle = 4
        while timeSlept < maxTime:
            taskStatus = cancelTask(url, auth, taskID, True)
            if not (taskStatus in ['WAITING','RUNNING']):
                break
            if verbose > 0:
                print "Time: "+str(timeSlept)+"/"+str(maxTime)
            time.sleep(sleepCycle)
            timeSlept += sleepCycle

    if verbose > 0:
        print "Task "+str(taskID)+" is done"

def downloadFromOSM(verbose, url, auth, transactionId, update, mapping, bbox):
    if verbose>0:
        print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': update, 'mapping': mapping, 'bbox': bbox, 'transactionId':transactionId}
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if verbose > 0:
        if response['task']['status'] == 'WAITING':
            print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if verbose > 0:
        if response['task']['status'] == 'RUNNING':
            print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def getRepoID(geoserver, auth, workspace, datastore):
    params = {}
    url = geoserver+"rest/workspaces/"+workspace+"/datastores/"+datastore+".json"
    request = make_request(url=url, params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    repoID = None
    for entry in response['dataStore']['connectionParameters']['entry']:
        if entry['@key'] == 'geogig_repository':
            repoID = entry['$']
            break

    return repoID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def parse_bbox(extent):
    file_extent = "/opt/cybergis-osm-mappings.git/extents/"+extent.replace(":","/")+".txt"
    bbox = None
    with open (file_extent, "r") as f:
        bbox = f.read().replace('\n', '').replace(' ',',')
    return bbox

def parse_mapping(ns_mapping):
    file_mapping = "/opt/cybergis-osm-mappings.git/mappings/"+ns_mapping.replace(":","/")+".json"
    return file_mapping

def run(args):
    #==#
    verbose = args.verbose
    #==#
    geoserver = parse_url(args.geoserver)
    repo = args.repo
    datastore = args.datastore
    workspace = args.workspace
    timeout = args.timeout or 30
    #==#
    authorname = args.authorname
    authoremail = args.authoremail
    #==#
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))
    #==#
    update = args.update in ["1","y","t","true"]
    bbox = parse_bbox(args.extent)
    mapping = parse_mapping(args.mapping)
    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    if repo:
        pass
    elif geoserver and workspace and datastore:
        repo = getRepoID(geoserver, auth, workspace, datastore)
    else:
        print "You need to include the repo id or the datastore name to sync"
    #==#
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'
    #==#
    if update:
        pass
    elif bbox and mapping:
       pass
    else:
        return "Update is false and no new data will be brought in because the extent and mapping aren't specified"

    transID = -1
    try:
        transID = beginTransaction(url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            branch = checkout(url_repo, auth, 'master', transID)
            if not branch:
                raise Exception('An error occurred when checking out master.  Cancelling task.')
            taskID = downloadFromOSM(verbose, url_repo, auth, transID, update, mapping, bbox)
        except Exception:
            taskID = -1
            endTransaction(url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(verbose, url_tasks, auth, taskID, timeout)

        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            checkout(url_repo, auth, 'master', transID)
        except Exception:
            pass

    try:
        endTransaction(url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')

parser.add_argument("update", default= 'true', help="true/false.  Update existing features only or download new features.  If false, extent and mapping are required.")

#
parser.add_argument("--workspace", help="The workspace of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--datastore", help="The name of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")

parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")
parser.add_argument("--extent", help="The extent of the OpenStreetMap extract. For example, basic:buildings_and_roads.")
parser.add_argument("--mapping", help="The mapping of the OpenStreetMap extract.  For example, dominican_republic:santo_domingo.")
parser.add_argument("--timeout", type=int, default=30, help="The number of seconds to wait for the osm download task to complete before cancelling.  Default is 30 seconds.")
parser.add_argument('--verbose', '-v', default=0, action='count', help="Print out intermediate status messages.")

args = parser.parse_args()
run(args)
<commit_msg>Update geogig_sync_osm.py<commit_after>from base64 import b64encode
from optparse import make_option
import json
import urllib
import urllib2
import argparse
import time

def make_request(url, params, auth=None):
    """
    Prepares a request from a url, params, and optionally authentication.
    """
    req = urllib2.Request(url + urllib.urlencode(params))

    if auth:
        req.add_header('AUTHORIZATION', 'Basic ' + auth)

    return urllib2.urlopen(req)

def beginTransaction(verbose, url, auth):
    if verbose > 0:
        print('Starting transaction...')
    params = {'output_format': 'JSON'}
    request = make_request(url=url+'beginTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("action failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on action: {0}".format(response['response']['error']))

    if verbose > 0:
        print('Transaction started.')
    transactionId = response['response']['Transaction']['ID']

    return transactionId;

def endTransaction(verbose, url, auth, cancel, transactionId):
    if verbose > 0:
        print('Ending transaction...')
    params = {'output_format': 'JSON', 'cancel': cancel, 'transactionId': transactionId}
    request = make_request(url=url+'endTransaction.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("EndTransaction failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    if not response['response']['success']:
        raise Exception("An error occurred on endTransaction: {0}".format(response['response']['error']))

    if verbose > 0:
        print('Transaction ended.')

def checkout(url, auth, branch, transactionId):
    print "Checking out "+branch+" branch..."
    params = {'output_format': 'JSON', 'branch': branch, 'transactionId':transactionId}
    request = make_request(url=url+'checkout.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Checkout for branch "+branch+" failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    #print response
    if response['response']['success']:
        newBranch = response['response']['NewTarget']
        print "Checked out "+newBranch+' branch.'
        return newBranch
    else:
        print "----"
        print "Checkout for "+branch+" branch failed."
        print "Error Message: "+response['response']['error']
        return None

def pollTask(url, auth, taskID):
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json', params=params, auth=auth)

    #print request.getcode()
    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    status = response['task']['status']

    progress = None
    try:
        progress = response['task']['progress']
    except:
        progress = None

    result = None
    try:
        result = response['task']['result']
    except:
        result = None

    errorMessage = None
    try:
        errorMessage = response['task']['error']['message']
    except:
        errorMessage = None

    return status, progress, result, errorMessage

def printTaskStatus(taskID, status, progress, result, errorMessage):

    if status == "RUNNING":
        print "----"
        print "++Task "+str(taskID)+" is running."
        if progress:
            progress_task = progress['task']
            if progress_task == 'Committing features...':
                print progress_task
            elif progress_task == 'Importing into GeoGig repo...':
                print progress_task+' '+str(progress['amount'])+' entities imported'
            else:
                print "Current Step: "+str(progress['task'])
                print "Entities Processed: "+str(progress['amount'])
        if result:
            print "Entities Processed: "+str(result['OSMReport']['processedEntities'])
    elif status == "FAILED":
        print "----"
        print "++Task "+str(taskID)+" failed with error message: "+errorMessage+"."
    elif status == "FINISHED":
        print "----"
        print "++Task "+str(taskID)+" is finished."
        if result:
            print "Entities Processed: "+str(result['OSMReport']['processedEntities'])
    elif status == "CANCELLED":
        print "----"
        print response
        print "++Task "+str(taskID)+" was cancelled."
        print "Entities Processed: "+str(result['OSMReport']['processedEntities'])
    else:
        print "----"
        print "++Task "+str(taskID)+" is "+status+"."

def cancelTask(url, auth, taskID, printStatus):
    print('Downloading from OpenStreetMap ...')
    params = {}
    request = make_request(url=url+'/'+str(taskID)+'.json?cancel=true', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Could not cancel task: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskStatus = response['task']['status']

    if printStatus:
        if taskStatus == "CANCELLED":
            print "++Task "+str(taskID)+" was cancelled."
        else:
            print "++Error.  Could not cancel task "+str(taskID)+".  Task is "+str(taskStatus)+"."

    return taskStatus;

def waitOnTask(verbose, url, auth, taskID, timeout):
    maxTime = timeout
    timeSlept = 0
    sleepCycle = 5
    taskStatus = None

    print "----------------------------------"
    print "Waiting for task "+str(taskID)+"..."
    print "Maximum wait time is "+str(maxTime)+" seconds."
    while timeSlept < maxTime:
        taskStatus, taskProgress, taskResult, errorMessage = pollTask(url, auth, taskID)
        if verbose > 0:
            printTaskStatus(taskID, taskStatus, taskProgress, taskResult, errorMessage)
        if not (taskStatus in ['WAITING','RUNNING']):
            break
        if verbose > 0:
            print "Time: "+str(timeSlept)+"/"+str(maxTime)
        time.sleep(sleepCycle)
        timeSlept += sleepCycle

    if taskStatus in ['WAITING','RUNNING']:
        print "Task "+str(taskID)+" timed out after "+str(timeSlept)+" seconds."
        print "Attempting to cancel task "+str(taskID)
        maxTime = 60
        timeSlept = 0
        sleepCycle = 4
        while timeSlept < maxTime:
            taskStatus = cancelTask(url, auth, taskID, True)
            if not (taskStatus in ['WAITING','RUNNING']):
                break
            if verbose > 0:
                print "Time: "+str(timeSlept)+"/"+str(maxTime)
            time.sleep(sleepCycle)
            timeSlept += sleepCycle

    if verbose > 0:
        print "Task "+str(taskID)+" is done"

def downloadFromOSM(verbose, url, auth, transactionId, update, mapping, bbox):
    if verbose>0:
        print('Downloading from OpenStreetMap ...')
    params = {'output_format': 'JSON', 'update': update, 'mapping': mapping, 'bbox': bbox, 'transactionId':transactionId}
    request = make_request(url=url+'osm/download.json?', params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("OSM Download failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())

    taskID = response['task']['id']

    if response['task']['status'] == 'FAILED':
        raise Exception("An error occurred when pulling new data from OSM: {0}".format(response['task']['status']))

    if verbose > 0:
        if response['task']['status'] == 'WAITING':
            print('Download from OpenStreetMap is waiting to be processed.  Task ID is '+str(taskID)+'.')

    if verbose > 0:
        if response['task']['status'] == 'RUNNING':
            print('Download from OpenStreetMap is being processed.  Task ID is '+str(taskID)+'.')

    return taskID;

def getRepoID(geoserver, auth, workspace, datastore):
    params = {}
    url = geoserver+"rest/workspaces/"+workspace+"/datastores/"+datastore+".json"
    request = make_request(url=url, params=params, auth=auth)

    if request.getcode() != 200:
        raise Exception("Get Task Status Failed: Status Code {0}".format(request.getcode()))

    response = json.loads(request.read())
    repoID = None
    for entry in response['dataStore']['connectionParameters']['entry']:
        if entry['@key'] == 'geogig_repository':
            repoID = entry['$']
            break

    return repoID;

def parse_url(url):

    if (url is None) or len(url) == 0:
        return None

    index = url.rfind('/')

    if index != (len(url)-1):
        url += '/'

    return url

def parse_bbox(extent):
    file_extent = "/opt/cybergis-osm-mappings.git/extents/"+extent.replace(":","/")+".txt"
    bbox = None
    with open (file_extent, "r") as f:
        bbox = f.read().replace('\n', '').replace(' ',',')
    return bbox

def parse_mapping(ns_mapping):
    file_mapping = "/opt/cybergis-osm-mappings.git/mappings/"+ns_mapping.replace(":","/")+".json"
    return file_mapping

def run(args):
    #==#
    verbose = args.verbose
    #==#
    geoserver = parse_url(args.geoserver)
    repo = args.repo
    datastore = args.datastore
    workspace = args.workspace
    timeout = args.timeout or 30
    #==#
    authorname = args.authorname
    authoremail = args.authoremail
    #==#
    auth = None
    if args.username and args.password:
      auth = b64encode('{0}:{1}'.format(args.username, args.password))
    #==#
    update = args.update in ["1","y","t","true"]
    bbox = parse_bbox(args.extent)
    mapping = parse_mapping(args.mapping)
    print "=================================="
    print "#==#"
    print "CyberGIS Script / geogig_sync_osm.py"
    print "Downloading Updates from OpenStreetMap"
    print "#==#"

    if repo:
        pass
    elif geoserver and workspace and datastore:
        repo = getRepoID(geoserver, auth, workspace, datastore)
    else:
        print "You need to include the repo id or the datastore name to sync"
    #==#
    url_repo = geoserver+'geogig/'+repo+'/'
    url_tasks = geoserver+'geogig/tasks'
    #==#
    if update:
        pass
    elif bbox and mapping:
       pass
    else:
        return "Update is false and no new data will be brought in because the extent and mapping aren't specified"

    transID = -1
    try:
        transID = beginTransaction(verbose,url_repo, auth)
    except Exception:
        transID = -1
        raise

    if transID != -1:
        taskID = -1
        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            branch = checkout(url_repo, auth, 'master', transID)
            if not branch:
                raise Exception('An error occurred when checking out master.  Cancelling task.')
            taskID = downloadFromOSM(verbose, url_repo, auth, transID, update, mapping, bbox)
        except Exception:
            taskID = -1
            endTransaction(verbose,url_repo, auth, True, transID)
            raise

        if taskID != -1:
            waitOnTask(verbose, url_tasks, auth, taskID, timeout)

        #==#
        #Checkout master branch.  See: https://github.com/boundlessgeo/GeoGig/issues/788
        try:
            checkout(url_repo, auth, 'master', transID)
        except Exception:
            pass

    try:
        endTransaction(verbose,url_repo, auth, False, transID)
    except Exception:
        pass

    print "=================================="

parser = argparse.ArgumentParser(description='Synchronize GeoGig repository with OpenStreetMap (OSM)')

parser.add_argument("update", default= 'true', help="true/false.  Update existing features only or download new features.  If false, extent and mapping are required.")

#
parser.add_argument("--workspace", help="The workspace of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--datastore", help="The name of the GeoServer data store of the GeoGig repository you want to sync.")
parser.add_argument("--repo", help="The GeoServer id of the GeoGig repository you want to sync.")

parser.add_argument("--geoserver", help="The url of the GeoServer servicing the GeoGig repository.")
parser.add_argument("--username", help="The username to use for basic auth requests.")
parser.add_argument("--password", help="The password to use for basic auth requests.")
parser.add_argument("--authorname", help="The author name to use when merging non-conflicting branches.")
parser.add_argument("--authoremail", help="The author email to use when merging non-conflicting branches.")
parser.add_argument("--extent", help="The extent of the OpenStreetMap extract. For example, basic:buildings_and_roads.")
parser.add_argument("--mapping", help="The mapping of the OpenStreetMap extract.  For example, dominican_republic:santo_domingo.")
parser.add_argument("--timeout", type=int, default=30, help="The number of seconds to wait for the osm download task to complete before cancelling.  Default is 30 seconds.")
parser.add_argument('--verbose', '-v', default=0, action='count', help="Print out intermediate status messages.")

args = parser.parse_args()
run(args)
<|endoftext|>


<commit_before>import argparse
import datetime
import json
import os
import re

from os import path

from bottle import (
        abort,
        auth_basic,
        default_app,
        get,
        hook,
        post,
        redirect,
        request,
        route,
        run,
        static_file,
        template,
    )

DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"

config = None

def check_auth(username, password):
    return username == config['username'] and \
            password == config['password']

@get('/')
def index():
    all_lists = get_all_list_names()
    return template("templates/index.tpl", title="index", lists=sorted(all_lists))

@get('/l/<name:re:[a-z0-9-_]+>')
def list_view(name):
    validate_list(name)

    list_contents = get_list(name)
    return template("templates/list.tpl", list_name=name, list_contents=list_contents)

@post('/l/<name:re:[a-z0-9-_]+>/update')
@auth_basic(check_auth)
def list_update(name):
    list_item_text = request.forms.get('list_item_text', None)

    if list_item_text is None:
        abort(400, "List item not specified")

    validate_list(name)

    new_item = {
        "text": list_item_text,
        "creation_date": datetime.datetime.now().isoformat(),
        "is_checked": False
    }

    list_contents = get_list(name)
    list_contents.append(new_item)

    update_list(name, list_contents)

    redirect("/l/" + name)

@get('/l/<name:re:[a-z0-9-_]+>/delete/<index:int>')
@auth_basic(check_auth)
def list_item_delete(name, index):
    validate_list(name)

    list_contents = get_list(name)
    del list_contents[index]

    update_list(name, list_contents)

    redirect("/l/" + name)

@get('/l/<name:re:[a-z0-9-_]+>/check/<index:int>')
@auth_basic(check_auth)
def list_item_check(name, index):
    validate_list(name)

    list_contents = get_list(name)

    list_contents[index]['is_checked'] = not list_contents[index]['is_checked']

    update_list(name, list_contents)

    redirect("/l/" + name)

@get('/l/<name:re:[a-z0-9-_]+>/clearchecked')
@auth_basic(check_auth)
def list_clear_checked(name):
    validate_list(name)

    new_list = [x for x in get_list(name) if not x['is_checked']]
    update_list(name, new_list)

    redirect("/l/" + name)

@post('/s/add')
@auth_basic(check_auth)
def list_add():
    name = request.forms.get('list_name', None).lower()

    if name is None:
        abort(400, "List name not specified")

    all_lists = get_all_list_names()
    if name in all_lists:
        abort(400, "List already exists")

    update_list(name, [])

    redirect("/l/" + name)

@post('/s/delete')
@auth_basic(check_auth)
def list_delete():
    name = request.forms.get('list_name', None).lower()
    if name is None:
        abort(400, "List name not specified")

    all_lists = get_all_list_names()
    if name not in all_lists:
        abort(400, "List does not exists")

    delete_list(name):
    redirect("/")

def get_all_list_names():
    return [x.split(".")[0] for x in os.listdir(config['data_path'])]

def update_list(name, list_contents):
    with open(get_fileloc(name), "w") as f:
        json.dump(list_contents, f, indent=4, sort_keys=True)

def get_list(name):
    with open(get_fileloc(name), "r") as f:
        return json.load(f)

def delete_list(name):
    os.remove(get_fileloc(name))

def get_fileloc(name):
    filename = name + ".json"
    fileloc = path.join(config['data_path'], filename)

    return fileloc

def validate_list(name):
    if re.match(r"[a-z0-9-_]+", name) is None:
        abort(400, "Invalid list name")

    all_lists = get_all_list_names()
    if name not in all_lists:
        abort(404, "List not found")

@route('/static/<path:path>')
def static(path):
    return static_file(path, root="./static")

# remove
@hook('before_request')
def strip_path():
    request.environ['PATH_INFO'] = request.environ['PATH_INFO'].rstrip('/')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='starts a lists server')
    parser.add_argument('--config', help='specifies the config file location (default: ./config.json)',
                            default="./config.json")

    args = parser.parse_args()

    with open(args.config) as f:
        config = json.load(f)

    run(host='0.0.0.0', port=config['port'])

app = default_app()
<commit_msg>fixed typo bug<commit_after>import argparse
import datetime
import json
import os
import re

from os import path

from bottle import (
        abort,
        auth_basic,
        default_app,
        get,
        hook,
        post,
        redirect,
        request,
        route,
        run,
        static_file,
        template,
    )

DATE_FORMAT = "%Y-%m-%dT%H:%M:%S"

config = None

def check_auth(username, password):
    return username == config['username'] and \
            password == config['password']

@get('/')
def index():
    all_lists = get_all_list_names()
    return template("templates/index.tpl", title="index", lists=sorted(all_lists))

@get('/l/<name:re:[a-z0-9-_]+>')
def list_view(name):
    validate_list(name)

    list_contents = get_list(name)
    return template("templates/list.tpl", list_name=name, list_contents=list_contents)

@post('/l/<name:re:[a-z0-9-_]+>/update')
@auth_basic(check_auth)
def list_update(name):
    list_item_text = request.forms.get('list_item_text', None)

    if list_item_text is None:
        abort(400, "List item not specified")

    validate_list(name)

    new_item = {
        "text": list_item_text,
        "creation_date": datetime.datetime.now().isoformat(),
        "is_checked": False
    }

    list_contents = get_list(name)
    list_contents.append(new_item)

    update_list(name, list_contents)

    redirect("/l/" + name)

@get('/l/<name:re:[a-z0-9-_]+>/delete/<index:int>')
@auth_basic(check_auth)
def list_item_delete(name, index):
    validate_list(name)

    list_contents = get_list(name)
    del list_contents[index]

    update_list(name, list_contents)

    redirect("/l/" + name)

@get('/l/<name:re:[a-z0-9-_]+>/check/<index:int>')
@auth_basic(check_auth)
def list_item_check(name, index):
    validate_list(name)

    list_contents = get_list(name)

    list_contents[index]['is_checked'] = not list_contents[index]['is_checked']

    update_list(name, list_contents)

    redirect("/l/" + name)

@get('/l/<name:re:[a-z0-9-_]+>/clearchecked')
@auth_basic(check_auth)
def list_clear_checked(name):
    validate_list(name)

    new_list = [x for x in get_list(name) if not x['is_checked']]
    update_list(name, new_list)

    redirect("/l/" + name)

@post('/s/add')
@auth_basic(check_auth)
def list_add():
    name = request.forms.get('list_name', None).lower()

    if name is None:
        abort(400, "List name not specified")

    all_lists = get_all_list_names()
    if name in all_lists:
        abort(400, "List already exists")

    update_list(name, [])

    redirect("/l/" + name)

@post('/s/delete')
@auth_basic(check_auth)
def list_delete():
    name = request.forms.get('list_name', None).lower()
    if name is None:
        abort(400, "List name not specified")

    all_lists = get_all_list_names()
    if name not in all_lists:
        abort(400, "List does not exists")

    delete_list(name)
    redirect("/")

def get_all_list_names():
    return [x.split(".")[0] for x in os.listdir(config['data_path'])]

def update_list(name, list_contents):
    with open(get_fileloc(name), "w") as f:
        json.dump(list_contents, f, indent=4, sort_keys=True)

def get_list(name):
    with open(get_fileloc(name), "r") as f:
        return json.load(f)

def delete_list(name):
    os.remove(get_fileloc(name))

def get_fileloc(name):
    filename = name + ".json"
    fileloc = path.join(config['data_path'], filename)

    return fileloc

def validate_list(name):
    if re.match(r"[a-z0-9-_]+", name) is None:
        abort(400, "Invalid list name")

    all_lists = get_all_list_names()
    if name not in all_lists:
        abort(404, "List not found")

@route('/static/<path:path>')
def static(path):
    return static_file(path, root="./static")

# remove
@hook('before_request')
def strip_path():
    request.environ['PATH_INFO'] = request.environ['PATH_INFO'].rstrip('/')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='starts a lists server')
    parser.add_argument('--config', help='specifies the config file location (default: ./config.json)',
                            default="./config.json")

    args = parser.parse_args()

    with open(args.config) as f:
        config = json.load(f)

    run(host='0.0.0.0', port=config['port'])

app = default_app()
<|endoftext|>


<commit_before>from rflib.defs import *
from binascii import *
from rflib.types.Match import *
from rflib.types.Action import *
from rflib.types.Option import *

def create_default_flow_mod(dp, cookie=0, cookie_mask=0, table_id=0,
                            command=None, idle_timeout=0, hard_timeout=0,
                            priority=PRIORITY_LOWEST,
                            buffer_id=0xffffffff, match=None, actions=None,
                            inst_type=None, out_port=None, out_group=None,
                            flags=0, inst=[]):

  if command is None:
    command = dp.ofproto.OFPFC_ADD

  if inst is []:
    if inst_type is None:
      inst_type = dp.ofproto.OFPIT_APPLY_ACTIONS

    if actions is not None:
      inst = [dp.ofproto_parser.OFPInstructionActions(inst_type, actions)]

  if match is None:
    match = dp.ofproto_parser.OFPMatch()

  if out_port is None:
    out_port = dp.ofproto.OFPP_ANY

  if out_group is None:
    out_group = dp.ofproto.OFPG_ANY

  return dp.ofproto_parser.OFPFlowMod(dp, cookie, cookie_mask,
                                      table_id, command,
                                      idle_timeout, hard_timeout,
                                      priority, buffer_id,
                                      out_port, out_group,
                                      flags, match, inst)

def create_flow_mod(dp, mod, matches, actions, options):
  flow_mod = create_default_flow_mod(dp)
  add_command(flow_mod, mod)
  add_matches(flow_mod, matches)
  add_actions(flow_mod, actions)
  add_options(flow_mod, options)
  return flow_mod

def add_command(flow_mod, mod):
  if mod == RMT_ADD:
    pass
  elif mod == RMT_DELETE:
    flow_mod.command = flow_mod.datapath.ofproto.OFPFC_DELETE_STRICT

def add_matches(flow_mod, matches):
  for match in matches:
    if match['type'] == RFMT_IPV4:
      value = bin_to_int(match['value'])
      addr = value >> 32
      mask = value & ((1 << 32) - 1)
      flow_mod.match.set_dl_type(ETHERTYPE_IP)
      flow_mod.match.set_ipv4_dst_masked(addr, mask)
    elif match['type'] == RFMT_IPV6:
      v = match['value']
      addr = tuple((ord(v[i]) << 8) | ord(v[i + 1]) for i in range(0, 16, 2))
      mask = tuple((ord(v[i]) << 8) | ord(v[i + 1]) for i in range(16, 32, 2))
      flow_mod.match.set_dl_type(ETHERTYPE_IPV6)
      flow_mod.match.set_ipv6_dst_masked(addr, mask)
    elif match['type'] == RFMT_ETHERNET:
      flow_mod.match.set_dl_dst(match['value'])
    elif match['type'] == RFMT_ETHERTYPE:
      flow_mod.match.set_dl_type(bin_to_int(match['value']))
    elif match['type'] == RFMT_NW_PROTO:
      flow_mod.match.set_ip_proto(bin_to_int(match['value']))
    elif match['type'] == RFMT_TP_SRC:
      flow_mod.match.set_ip_proto(IPPROTO_TCP)
      flow_mod.match.set_tcp_src(bin_to_int(match['value']))
    elif match['type'] == RFMT_TP_DST:
      flow_mod.match.set_ip_proto(IPPROTO_TCP)
      flow_mod.match.set_tcp_dst(bin_to_int(match['value']))

def add_actions(flow_mod, action_tlvs):
  parser = flow_mod.datapath.ofproto_parser
  ofproto = flow_mod.datapath.ofproto
  actions = []
  for action in action_tlvs:
    if action['type'] == RFAT_OUTPUT:
      port = bin_to_int(action['value'])
      a = parser.OFPActionOutput(port, ofproto.OFPCML_MAX)
      actions.append(a)
    elif action['type'] == RFAT_SET_ETH_SRC:
      srcMac = action['value']
      src = parser.OFPMatchField.make(ofproto.OXM_OF_ETH_SRC, srcMac)
      actions.append(parser.OFPActionSetField(src))
    elif action['type'] == RFAT_SET_ETH_DST:
      dstMac = action['value']
      dst = parser.OFPMatchField.make(ofproto.OXM_OF_ETH_DST, dstMac)
      actions.append(parser.OFPActionSetField(dst))
  inst = parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)
  flow_mod.instructions = [inst]

def add_options(flow_mod, options):
  for option in options:
    if option['type'] == RFOT_PRIORITY:
      flow_mod.priority = bin_to_int(option['value'])
    elif option['type'] == RFOT_IDLE_TIMEOUT:
      flow_mod.idle_timeout = bin_to_int(option['value'])
    elif option['type'] == RFOT_HARD_TIMEOUT:
      flow_mod.hard_timeout = bin_to_int(option['value'])
    elif option['type'] == RFOT_CT_ID:
      pass

def send_pkt_out(dp, port, msg):
  actions = [dp.ofproto_parser.OFPActionOutput(port, len(msg)), ]
  dp.send_packet_out(buffer_id=0xffffffff, in_port=dp.ofproto.OFPP_ANY,
                     actions=actions, data=msg)
<commit_msg>Log RouteMod TLVs that we cannot parse<commit_after>import logging

from rflib.defs import *
from binascii import *
from rflib.types.Match import *
from rflib.types.Action import *
from rflib.types.Option import *

log = logging.getLogger('ryu.app.rfproxy')

def create_default_flow_mod(dp, cookie=0, cookie_mask=0, table_id=0,
                            command=None, idle_timeout=0, hard_timeout=0,
                            priority=PRIORITY_LOWEST,
                            buffer_id=0xffffffff, match=None, actions=None,
                            inst_type=None, out_port=None, out_group=None,
                            flags=0, inst=[]):

  if command is None:
    command = dp.ofproto.OFPFC_ADD

  if inst is []:
    if inst_type is None:
      inst_type = dp.ofproto.OFPIT_APPLY_ACTIONS

    if actions is not None:
      inst = [dp.ofproto_parser.OFPInstructionActions(inst_type, actions)]

  if match is None:
    match = dp.ofproto_parser.OFPMatch()

  if out_port is None:
    out_port = dp.ofproto.OFPP_ANY

  if out_group is None:
    out_group = dp.ofproto.OFPG_ANY

  return dp.ofproto_parser.OFPFlowMod(dp, cookie, cookie_mask,
                                      table_id, command,
                                      idle_timeout, hard_timeout,
                                      priority, buffer_id,
                                      out_port, out_group,
                                      flags, match, inst)

def create_flow_mod(dp, mod, matches, actions, options):
  flow_mod = create_default_flow_mod(dp)
  add_command(flow_mod, mod)
  add_matches(flow_mod, matches)
  add_actions(flow_mod, actions)
  add_options(flow_mod, options)
  return flow_mod

def add_command(flow_mod, mod):
  if mod == RMT_ADD:
    pass
  elif mod == RMT_DELETE:
    flow_mod.command = flow_mod.datapath.ofproto.OFPFC_DELETE_STRICT

def add_matches(flow_mod, matches):
  for match in matches:
    if match['type'] == RFMT_IPV4:
      value = bin_to_int(match['value'])
      addr = value >> 32
      mask = value & ((1 << 32) - 1)
      flow_mod.match.set_dl_type(ETHERTYPE_IP)
      flow_mod.match.set_ipv4_dst_masked(addr, mask)
    elif match['type'] == RFMT_IPV6:
      v = match['value']
      addr = tuple((ord(v[i]) << 8) | ord(v[i + 1]) for i in range(0, 16, 2))
      mask = tuple((ord(v[i]) << 8) | ord(v[i + 1]) for i in range(16, 32, 2))
      flow_mod.match.set_dl_type(ETHERTYPE_IPV6)
      flow_mod.match.set_ipv6_dst_masked(addr, mask)
    elif match['type'] == RFMT_ETHERNET:
      flow_mod.match.set_dl_dst(match['value'])
    elif match['type'] == RFMT_ETHERTYPE:
      flow_mod.match.set_dl_type(bin_to_int(match['value']))
    elif match['type'] == RFMT_NW_PROTO:
      flow_mod.match.set_ip_proto(bin_to_int(match['value']))
    elif match['type'] == RFMT_TP_SRC:
      flow_mod.match.set_ip_proto(IPPROTO_TCP)
      flow_mod.match.set_tcp_src(bin_to_int(match['value']))
    elif match['type'] == RFMT_TP_DST:
      flow_mod.match.set_ip_proto(IPPROTO_TCP)
      flow_mod.match.set_tcp_dst(bin_to_int(match['value']))
    elif match.optional():
        log.info("Dropping unsupported Match (type: %s)" % match._type)
    else:
        log.warning("Failed to serialise Match (type: %s)" % match._type)
        return

def add_actions(flow_mod, action_tlvs):
  parser = flow_mod.datapath.ofproto_parser
  ofproto = flow_mod.datapath.ofproto
  actions = []
  for action in action_tlvs:
    if action['type'] == RFAT_OUTPUT:
      port = bin_to_int(action['value'])
      a = parser.OFPActionOutput(port, ofproto.OFPCML_MAX)
      actions.append(a)
    elif action['type'] == RFAT_SET_ETH_SRC:
      srcMac = action['value']
      src = parser.OFPMatchField.make(ofproto.OXM_OF_ETH_SRC, srcMac)
      actions.append(parser.OFPActionSetField(src))
    elif action['type'] == RFAT_SET_ETH_DST:
      dstMac = action['value']
      dst = parser.OFPMatchField.make(ofproto.OXM_OF_ETH_DST, dstMac)
      actions.append(parser.OFPActionSetField(dst))
    elif action.optional():
        log.info("Dropping unsupported Action (type: %s)" % action._type)
    else:
        log.warning("Failed to serialise Action (type: %s)" % action._type)
        return
  inst = parser.OFPInstructionActions(ofproto.OFPIT_APPLY_ACTIONS, actions)
  flow_mod.instructions = [inst]

def add_options(flow_mod, options):
  for option in options:
    if option['type'] == RFOT_PRIORITY:
      flow_mod.priority = bin_to_int(option['value'])
    elif option['type'] == RFOT_IDLE_TIMEOUT:
      flow_mod.idle_timeout = bin_to_int(option['value'])
    elif option['type'] == RFOT_HARD_TIMEOUT:
      flow_mod.hard_timeout = bin_to_int(option['value'])
    elif option['type'] == RFOT_CT_ID:
      pass
    elif option.optional():
        log.info("Dropping unsupported Option (type: %s)" % option._type)
    else:
        log.warning("Failed to serialise Option (type: %s)" % option._type)
        return

def send_pkt_out(dp, port, msg):
  actions = [dp.ofproto_parser.OFPActionOutput(port, len(msg)), ]
  dp.send_packet_out(buffer_id=0xffffffff, in_port=dp.ofproto.OFPP_ANY,
                     actions=actions, data=msg)
<|endoftext|>


<commit_before>###############################################################################
##
##  hardcode_haml - Haml for hardcore coders
##  Copyright (C) 2010  Thammi
##
##  This program is free software: you can redistribute it and/or modify
##  it under the terms of the GNU Affero General Public License as published by
##  the Free Software Foundation, either version 3 of the License, or
##  (at your option) any later version.
##
##  This program is distributed in the hope that it will be useful,
##  but WITHOUT ANY WARRANTY; without even the implied warranty of
##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##  GNU Affero General Public License for more details.
##
##  You should have received a copy of the GNU Affero General Public License
##  along with this program.  If not, see <http://www.gnu.org/licenses/>.
##
###############################################################################

import re

from cpp import CppWriter
from c import CWriter

class ParserException(Exception):

    def __init__(self, msg, line=None):
        self.msg = msg
        self.line = line

    def __str__(self):
        if self.line == None:
            return self.msg
        else:
            return "[at line %i] %s" % (self.line, self.msg)

class XmlTag:

    def __init__(self, data, line=None):
        self.attrs = {}
        self.name = 'div'
        self.content = None
        self.line = line

        self.data = data

        self.parse_name()
        self.parse_attrs()
        self.parse_content()

    def add_attr(self, key, value):
        attrs = self.attrs

        if key in attrs:
            attrs[key].append(value)
        else:
            attrs[key] = [value]

    def consume_regex(self, regex):
        match = re.match(regex, self.data)

        if match:
            res = match.group()
            self.data = self.data[len(res):]
            return res
        else:
            return ""

    def parse_name_part(self):
        return self.consume_regex('\w+')

    def parse_name(self):
        while self.data and self.data[0] not in [' ', '(', '{']:
            type_c = self.data[0]
            self.data = self.data[1:]

            if type_c == '%':
                self.name = self.parse_name_part()
            elif type_c == '#':
                self.add_attr('id', '"%s"' % self.parse_name_part())
            elif type_c == '.':
                self.add_attr('class', '"%s"' % self.parse_name_part())
            else:
                raise ParserException("Couldn't parse tag name", self.line)

    def parse_attrs(self):
        if self.data and self.data[0] == '(':
            self.data = self.data[1:]

            while self.data[0] != ')':
                self.consume_regex("\s*")
                key = self.consume_regex("\w+")
                self.consume_regex("\s*=\s*")
                value = self.consume_regex("[^)\s]+")

                if value and key:
                    self.add_attr(key, value)
                else:
                    raise ParserException("Failed to parse the attributes", self.line)

            self.data = self.data[1:]
        elif self.data and self.data[0] == '{':
            self.data = self.data[1:]

            while self.data[0] != '}':
                self.consume_regex("\s*")
                key = self.consume_regex("\w+")
                self.consume_regex("\s*(?:=>)\s*")
                value = self.consume_regex('[^,}]+')
                self.consume_regex("\s*,?")

                if value and key:
                    self.add_attr(key, value)
                else:
                    print self.data
                    raise ParserException("Failed to parse the attribute", self.line)

            self.data = self.data[1:]

    def parse_content(self):
        self.content = self.data

class Parser:

    def __init__(self, inp, out, readable=True):
        self.inp = inp
        self.out = out
        self.readable = readable

    def write_indent(self):
        if self.readable:
            self.out.write(" " * (self.indent * 2))

    def sync_stack(self, to):
        stack = self.stack
        while len(stack) > to:
            fun = stack.pop()
            if fun: fun()

    def count_indent(self, line):
        return len(re.match("^\s*", line).group()) / 2

    def backshift_display(self, line):
        self.indent -= 1
        self.indent_display(line)

    def indent_display(self, line, pre_indent=False):
        self.write_indent()
        self.display(line, pre_indent)
        self.out.write("\n")

    def display(self, line, pre_indent=False):
        if line.startswith('='):
            self.out.evaluate(line[1:].strip())
        else:
            self.out.write(line)

    def escape(self, line, pre_indent):
        self.out.write(line[1:])

    def execute(self, line, pre_indent):
        if pre_indent:
            self.out.block_exec(line[1:].strip())
            return self.out.close_block
        else:
            self.out.execute(line[1:].strip())

    def parse_tag(self, line, pre_indent):
        tag = XmlTag(line, self.line)

        self.write_indent()

        self.out.write("<%s" % tag.name)

        for key, values in tag.attrs.iteritems():
            self.out.write(' %s="' % key)
            for index, value in enumerate(values):
                if index > 0:
                    self.out.write(" ")

                self.out.evaluate(value)

            self.out.write('"')

        self.out.write(">")

        if pre_indent:
            if tag.content:
                raise ParserException("No content allowed for nested tags", self.line)

            self.indent += 1
            self.out.write("\n")
            return lambda: self.backshift_display("</%s>" % tag.name)
        else:
            self.display(tag.content.strip())
            self.out.write("</%s>\n" % tag.name)

    def comment(self, line, pre_indent):
        self.write_indent()
        self.out.write('<!--')

        if pre_indent:
            self.indent += 1
            self.out.write("\n")
            return lambda: self.backshift_display("-->")
        else:
            self.out.write(" %s -->\n" % line[1:].strip())

    def line_split(self, inp):
        last = None
        self.line = 0

        for new in inp:
            if last and last.strip():
                yield (last, self.count_indent(new) > self.count_indent(last))

            last = new
            self.line += 1

        if last and last.strip():
            yield (last, False)

    def parse(self):
        out = self.out
        readable = self.readable

        out.start()

        self.indent = 0
        self.stack = stack = []

        actions = {
                '#': self.parse_tag,
                '%': self.parse_tag,
                '.': self.parse_tag,
                '-': self.execute,
                '/': self.comment,
                '\\': self.escape,
                }

        for line, pre_indent in self.line_split(self.inp):
            if readable:
                self.out.comment('syncing the stack')

            self.sync_stack(self.count_indent(line))

            if readable:
                self.out.comment('Haml line %i' % self.line)

            content = line.strip()

            start = content[0]
            action = actions[start] if start in actions else self.indent_display

            pop_val = action(content, pre_indent)
            stack.append(pop_val)

        if readable:
            self.out.comment("End of Haml file, clearing stack")

        # clear the stack
        self.sync_stack(0)

        out.finish()

<commit_msg>Evaluating in display expressions (#{})<commit_after>###############################################################################
##
##  hardcode_haml - Haml for hardcore coders
##  Copyright (C) 2010  Thammi
##
##  This program is free software: you can redistribute it and/or modify
##  it under the terms of the GNU Affero General Public License as published by
##  the Free Software Foundation, either version 3 of the License, or
##  (at your option) any later version.
##
##  This program is distributed in the hope that it will be useful,
##  but WITHOUT ANY WARRANTY; without even the implied warranty of
##  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
##  GNU Affero General Public License for more details.
##
##  You should have received a copy of the GNU Affero General Public License
##  along with this program.  If not, see <http://www.gnu.org/licenses/>.
##
###############################################################################

import re

class ParserException(Exception):

    def __init__(self, msg, line=None):
        self.msg = msg
        self.line = line

    def __str__(self):
        if self.line == None:
            return self.msg
        else:
            return "[at line %i] %s" % (self.line, self.msg)

class XmlTag:

    def __init__(self, data, line=None):
        self.attrs = {}
        self.name = 'div'
        self.content = None
        self.line = line

        self.data = data

        self.parse_name()
        self.parse_attrs()
        self.parse_content()

    def add_attr(self, key, value):
        attrs = self.attrs

        if key in attrs:
            attrs[key].append(value)
        else:
            attrs[key] = [value]

    def consume_regex(self, regex):
        match = re.match(regex, self.data)

        if match:
            res = match.group()
            self.data = self.data[len(res):]
            return res
        else:
            return ""

    def parse_name_part(self):
        return self.consume_regex('\w+')

    def parse_name(self):
        while self.data and self.data[0] not in [' ', '(', '{']:
            type_c = self.data[0]
            self.data = self.data[1:]

            if type_c == '%':
                self.name = self.parse_name_part()
            elif type_c == '#':
                self.add_attr('id', '"%s"' % self.parse_name_part())
            elif type_c == '.':
                self.add_attr('class', '"%s"' % self.parse_name_part())
            else:
                raise ParserException("Couldn't parse tag name", self.line)

    def parse_attrs(self):
        if self.data and self.data[0] == '(':
            self.data = self.data[1:]

            while self.data[0] != ')':
                self.consume_regex("\s*")
                key = self.consume_regex("\w+")
                self.consume_regex("\s*=\s*")
                value = self.consume_regex("[^)\s]+")

                if value and key:
                    self.add_attr(key, value)
                else:
                    raise ParserException("Failed to parse the attributes", self.line)

            self.data = self.data[1:]
        elif self.data and self.data[0] == '{':
            self.data = self.data[1:]

            while self.data[0] != '}':
                self.consume_regex("\s*")
                key = self.consume_regex("\w+")
                self.consume_regex("\s*(?:=>)\s*")
                value = self.consume_regex('[^,}]+')
                self.consume_regex("\s*,?")

                if value and key:
                    self.add_attr(key, value)
                else:
                    print self.data
                    raise ParserException("Failed to parse the attribute", self.line)

            self.data = self.data[1:]

    def parse_content(self):
        self.content = self.data

class Parser:

    def __init__(self, inp, out, readable=True):
        self.inp = inp
        self.out = out
        self.readable = readable

    def write_indent(self):
        if self.readable:
            self.out.write(" " * (self.indent * 2))

    def sync_stack(self, to):
        stack = self.stack
        while len(stack) > to:
            fun = stack.pop()
            if fun: fun()

    def count_indent(self, line):
        return len(re.match("^\s*", line).group()) / 2

    def backshift_display(self, line):
        self.indent -= 1
        self.indent_display(line)

    def indent_display(self, line, pre_indent=False):
        self.write_indent()
        self.display(line, pre_indent)
        self.out.write("\n")

    def display(self, line, pre_indent=False):
        if line.startswith('='):
            self.out.evaluate(line[1:].strip())
        else:
            last = 0

            # find evaluation expressions
            while True:
                index = line.find('#{', last)

                if index == -1:
                    break

                end_index = line.find('}', index)

                if end_index == -1:
                    raise ParserException("Evaluation subexpression not closed.", self.line)

                # print everything before the expression
                self.out.write(line[last:index])

                # process the actual evaluation
                self.out.evaluate(line[index+2:end_index])

                last = end_index + 1

            # print the end
            self.out.write(line[last:])

    def escape(self, line, pre_indent):
        self.out.write(line[1:])

    def execute(self, line, pre_indent):
        if pre_indent:
            self.out.block_exec(line[1:].strip())
            return self.out.close_block
        else:
            self.out.execute(line[1:].strip())

    def parse_tag(self, line, pre_indent):
        tag = XmlTag(line, self.line)

        self.write_indent()

        self.out.write("<%s" % tag.name)

        for key, values in tag.attrs.iteritems():
            self.out.write(' %s="' % key)
            for index, value in enumerate(values):
                if index > 0:
                    self.out.write(" ")

                self.out.evaluate(value)

            self.out.write('"')

        self.out.write(">")

        if pre_indent:
            if tag.content:
                raise ParserException("No content allowed for nested tags", self.line)

            self.indent += 1
            self.out.write("\n")
            return lambda: self.backshift_display("</%s>" % tag.name)
        else:
            self.display(tag.content.strip())
            self.out.write("</%s>\n" % tag.name)

    def comment(self, line, pre_indent):
        self.write_indent()
        self.out.write('<!--')

        if pre_indent:
            self.indent += 1
            self.out.write("\n")
            return lambda: self.backshift_display("-->")
        else:
            self.out.write(" %s -->\n" % line[1:].strip())

    def line_split(self, inp):
        last = None
        self.line = 0

        for new in inp:
            if last and last.strip():
                yield (last, self.count_indent(new) > self.count_indent(last))

            last = new
            self.line += 1

        if last and last.strip():
            yield (last, False)

    def parse(self):
        out = self.out
        readable = self.readable

        out.start()

        self.indent = 0
        self.stack = stack = []

        actions = {
                '#': self.parse_tag,
                '%': self.parse_tag,
                '.': self.parse_tag,
                '-': self.execute,
                '/': self.comment,
                '\\': self.escape,
                }

        for line, pre_indent in self.line_split(self.inp):
            if readable:
                self.out.comment('syncing the stack')

            self.sync_stack(self.count_indent(line))

            if readable:
                self.out.comment('Haml line %i' % self.line)

            content = line.strip()

            start = content[0]
            action = actions[start] if start in actions else self.indent_display

            pop_val = action(content, pre_indent)
            stack.append(pop_val)

        if readable:
            self.out.comment("End of Haml file, clearing stack")

        # clear the stack
        self.sync_stack(0)

        out.finish()

<|endoftext|>


<commit_before>from pyramid.traversal import find_root
from uuid import UUID
from ..migrator import upgrade_step
import re

@upgrade_step('replicate', '', '3')
def replicate_0_3(value, system):
    # http://redmine.encodedcc.org/issues/1074
    context = system['context']
    root = find_root(context)
    if 'library' in value:
        item = root.get_by_uuid(value['library'])
        value['status'] = item.properties['status']
    else:
        value['status'] = 'in progress'
    # http://redmine.encodedcc.org/issues/1354

    if 'paired_ended' in value and 'read_length' not in value:
        del value['paired_ended']
<commit_msg>First upgrade library to get updated status<commit_after>from ..migrator import upgrade_step
from pyramid.traversal import find_root

@upgrade_step('replicate', '', '3')
def replicate_0_3(value, system):
    # http://redmine.encodedcc.org/issues/1074
    context = system['context']
    root = find_root(context)
    if 'library' in value:
        library = root.get_by_uuid(value['library']).upgrade_properties(finalize=False)
        value['status'] = library['status']
    else:
        value['status'] = 'in progress'

    # http://redmine.encodedcc.org/issues/1354
    if 'paired_ended' in value and 'read_length' not in value:
        del value['paired_ended']
<|endoftext|>