<commit_before>#
# Copyright 2015 Benjamin Kiessling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
"""
Accessors to the model repository on zenodo.
"""
from typing import Callable, Any
from contextlib import closing

from kraken.lib.exceptions import KrakenRepoException

import os
import json
import urllib
import pathlib
import logging
import requests

__all__ = ['get_model', 'get_description', 'get_listing', 'publish_model']

logger = logging.getLogger(__name__)

MODEL_REPO = 'https://zenodo.org/api/'
SUPPORTED_MODELS = set(['kraken_pytorch'])

def publish_model(model_file: [str, pathlib.Path] = None,
                  metadata: dict = None,
                  access_token: str = None,
                  callback: Callable[[int, int], Any] = lambda: None) -> str:
    """
    Publishes a model to the repository.

    Args:
        model_file: Path to read model from.
        metadata: Metadata dictionary
        access_token: Zenodo API access token
        callback: Function called for every 1024 octet chunk uploaded.
    """
    fp = open(model_fle, 'rb')
    total = len(model_file) + 4
    headers = {"Content-Type": "application/json"}
    r = requests.post(f'{MODEL_REPO}deposit/depositions',
                      params={'access_token': access_token}, json={},
                      headers=headers)
    r.raise_for_status()
    callback(total, 1)
    deposition_id = r.json()['id']
    data = {'filename': 'metadata.json'}
    files = {'file': ('metadata.json', json.dumps(metadata))}
    r = requests.post(f'{MODEL_REPO}deposit/depositions/{deposition_id}/files',
                      params={'access_token': access_token}, data=data,
                      files=files)
    r.raise_for_status()
    callback(total, 1)
    data = {'filename': metadata['name']}
    files = {'file': open(model_file, 'rb')}
    r = requests.post(f'{MODEL_REPO}deposit/depositions/{deposition_id}/files',
                      params={'access_token': access_token}, data=data,
                      files=files)
    r.raise_for_status()
    callback(total, 1)
    # fill zenodo metadata
    data = {'metadata': {
                        'title': metadata['summary'],
                        'upload_type': 'publication',
                        'publication_type': 'other',
                        'description': metadata['description'],
                        'creators': metadata['authors'],
                        'access_right': 'open',
                        'communities': [{'identifier': 'ocr_models'}],
                        'keywords': ['kraken_pytorch'],
                        'license': metadata['license']
                        }
            }
    # add link to training data to metadata
    if 'source' in metadata:
        data['metadata']['related_identifiers'] = [{'relation': 'isSupplementTo', 'identifier': metadata['source']}]
    r = requests.put(f'{MODEL_REPO}deposit/depositions/{deposition_id}',
                     params={'access_token': access_token},
                     data=json.dumps(data),
                     headers=headers)
    r.raise_for_status()
    callback(total, 1)
    r = requests.post(f'{MODEL_REPO}deposit/depositions/{deposition_id}/actions/publish',
                      params={'access_token': access_token})
    r.raise_for_status()
    callback(total, len(model_file))
    return r.json()['doi']

def get_model(model_id: str, path: str, callback: Callable[[int, int], Any] = lambda total, advance: None) -> str:
    """
    Retrieves a model and saves it to a path.

    Args:
        model_id (str): DOI of the model
        path (str): Destination to write model to.
        callback (func): Function called for every 1024 octet chunk received.

    Returns:
        The identifier the model can be called through on the command line.
        Will usually be the file name of the model.
    """
    logger.info(f'Saving model {model_id} to {path}')
    r = requests.get(f'{MODEL_REPO}records', params={'q': f'doi:"{model_id}"'})
    r.raise_for_status()
    callback(0, 0)
    resp = r.json()
    if resp['hits']['total'] != 1:
        logger.error(f'Found {resp["hits"]["total"]} models when querying for id \'{model_id}\'')
        raise KrakenRepoException(f'Found {resp["hits"]["total"]} models when querying for id \'{model_id}\'')

    metadata = resp['hits']['hits'][0]
    model_url = [x['links']['self'] for x in metadata['files'] if x['type'] == 'mlmodel'][0]
    # callable model identifier
    nat_id = os.path.basename(urllib.parse.urlparse(model_url).path)
    spath = os.path.join(path, nat_id)
    logger.debug(f'downloading model file {model_url} to {spath}')
    with closing(requests.get(model_url, stream=True)) as r:
        file_size = int(r.headers['Content-length'])
        with open(spath, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                callback(file_size, len(chunk))
                f.write(chunk)
    return nat_id

def get_description(model_id: str, callback: Callable[..., Any] = lambda: None) -> dict:
    """
    Fetches the metadata for a single model from the zenodo repository.

    Args:
        model_id (str): DOI of the model.
        callback (callable): Optional function called once per HTTP request.

    Returns:
        Dict
    """
    logger.info(f'Retrieving metadata for {model_id}')
    r = requests.get(f'{MODEL_REPO}records', params={'q': f'doi:"{model_id}"'})
    r.raise_for_status()
    callback()
    resp = r.json()
    if resp['hits']['total'] != 1:
        logger.error(f'Found {resp["hits"]["total"]} models when querying for id \'{model_id}\'')
        raise KrakenRepoException(f'Found {resp["hits"]["total"]} models when querying for id \'{model_id}\'')
    record = resp['hits']['hits'][0]
    metadata = record['metadata']
    if 'keywords' not in metadata:
        logger.error('No keywords included on deposit')
        raise KrakenRepoException('No keywords included on deposit.')
    model_type = SUPPORTED_MODELS.intersection(metadata['keywords'])
    if not model_type:
        msg = 'Unsupported model type(s): {}'.format(', '.join(metadata['keywords']))
        logger.error(msg)
        raise KrakenRepoException(msg)
    meta_json = None
    for file in record['files']:
        if file['key'] == 'metadata.json':
            callback()
            r = requests.get(file['links']['self'])
            r.raise_for_status()
            callback()
            try:
                meta_json = r.json()
            except Exception:
                msg = f'Metadata for \'{record["metadata"]["title"]}\' ({record["metadata"]["doi"]}) not in JSON format'
                logger.error(msg)
                raise KrakenRepoException(msg)
    if not meta_json:
        msg = 'Mo metadata.jsn found for \'{}\' ({})'.format(record['metadata']['title'], record['metadata']['doi'])
        logger.error(msg)
        raise KrakenRepoException(msg)
    # merge metadata.json into DataCite
    metadata.update({'graphemes': meta_json['graphemes'],
                     'summary': meta_json['summary'],
                     'script': meta_json['script'],
                     'link': record['links']['latest'],
                     'type': [x.split('_')[1] for x in model_type],
                     'accuracy': meta_json['accuracy']})
    return metadata

def get_listing(callback: Callable[[int, int], Any] = lambda total, advance: None) -> dict:
    """
    Fetches a listing of all kraken models from the zenodo repository.

    Args:
        callback (Callable): Function called after each HTTP request.

    Returns:
        Dict of models with each model.
    """
    logger.info('Retrieving model list')
    records = []
    r = requests.get('{}{}'.format(MODEL_REPO, 'records'), params={'communities': 'ocr_models'})
    r.raise_for_status()
    callback(1, 1)
    resp = r.json()
    if not resp['hits']['total']:
        logger.error('No models found in community \'ocr_models\'')
        raise KrakenRepoException('No models found in repository \'ocr_models\'')
    logger.debug('Total of {} records in repository'.format(resp['hits']['total']))
    total = resp['hits']['total']
    callback(total, 0)
    records.extend(resp['hits']['hits'])
    while 'next' in resp['links']:
        logger.debug('Fetching next page')
        r = requests.get(resp['links']['next'])
        r.raise_for_status()
        resp = r.json()
        logger.debug('Found {} new records'.format(len(resp['hits']['hits'])))
        records.extend(resp['hits']['hits'])
    logger.debug('Retrieving model metadata')
    models = {}
    # fetch metadata.jsn for each model
    for record in records:
        if 'keywords' not in record['metadata']:
            continue
        model_type = SUPPORTED_MODELS.intersection(record['metadata']['keywords'])
        if not model_type:
            continue
        for file in record['files']:
            if file['key'] == 'metadata.json':
                callback(total, 1)
                r = requests.get(file['links']['self'])
                r.raise_for_status()
                try:
                    metadata = r.json()
                except Exception:
                    msg = f'Metadata for \'{record["metadata"]["title"]}\' ({record["metadata"]["doi"]}) not in JSON format'
                    logger.error(msg)
                    raise KrakenRepoException(msg)
        # merge metadata.jsn into DataCite
        key = record['metadata']['doi']
        models[key] = record['metadata']
        models[key].update({'graphemes': metadata['graphemes'],
                            'summary': metadata['summary'],
                            'script': metadata['script'],
                            'link': record['links']['latest'],
                            'type': [x.split('_')[1] for x in model_type]})
    return models
<commit_msg>fix typo<commit_after>#
# Copyright 2015 Benjamin Kiessling
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing
# permissions and limitations under the License.
"""
Accessors to the model repository on zenodo.
"""
from typing import Callable, Any
from contextlib import closing

from kraken.lib.exceptions import KrakenRepoException

import os
import json
import urllib
import pathlib
import logging
import requests

__all__ = ['get_model', 'get_description', 'get_listing', 'publish_model']

logger = logging.getLogger(__name__)

MODEL_REPO = 'https://zenodo.org/api/'
SUPPORTED_MODELS = set(['kraken_pytorch'])

def publish_model(model_file: [str, pathlib.Path] = None,
                  metadata: dict = None,
                  access_token: str = None,
                  callback: Callable[[int, int], Any] = lambda: None) -> str:
    """
    Publishes a model to the repository.

    Args:
        model_file: Path to read model from.
        metadata: Metadata dictionary
        access_token: Zenodo API access token
        callback: Function called for every 1024 octet chunk uploaded.
    """
    fp = open(model_file, 'rb')
    total = len(model_file) + 4
    headers = {"Content-Type": "application/json"}
    r = requests.post(f'{MODEL_REPO}deposit/depositions',
                      params={'access_token': access_token}, json={},
                      headers=headers)
    r.raise_for_status()
    callback(total, 1)
    deposition_id = r.json()['id']
    data = {'filename': 'metadata.json'}
    files = {'file': ('metadata.json', json.dumps(metadata))}
    r = requests.post(f'{MODEL_REPO}deposit/depositions/{deposition_id}/files',
                      params={'access_token': access_token}, data=data,
                      files=files)
    r.raise_for_status()
    callback(total, 1)
    data = {'filename': metadata['name']}
    files = {'file': open(model_file, 'rb')}
    r = requests.post(f'{MODEL_REPO}deposit/depositions/{deposition_id}/files',
                      params={'access_token': access_token}, data=data,
                      files=files)
    r.raise_for_status()
    callback(total, 1)
    # fill zenodo metadata
    data = {'metadata': {
                        'title': metadata['summary'],
                        'upload_type': 'publication',
                        'publication_type': 'other',
                        'description': metadata['description'],
                        'creators': metadata['authors'],
                        'access_right': 'open',
                        'communities': [{'identifier': 'ocr_models'}],
                        'keywords': ['kraken_pytorch'],
                        'license': metadata['license']
                        }
            }
    # add link to training data to metadata
    if 'source' in metadata:
        data['metadata']['related_identifiers'] = [{'relation': 'isSupplementTo', 'identifier': metadata['source']}]
    r = requests.put(f'{MODEL_REPO}deposit/depositions/{deposition_id}',
                     params={'access_token': access_token},
                     data=json.dumps(data),
                     headers=headers)
    r.raise_for_status()
    callback(total, 1)
    r = requests.post(f'{MODEL_REPO}deposit/depositions/{deposition_id}/actions/publish',
                      params={'access_token': access_token})
    r.raise_for_status()
    callback(total, len(model_file))
    return r.json()['doi']

def get_model(model_id: str, path: str, callback: Callable[[int, int], Any] = lambda total, advance: None) -> str:
    """
    Retrieves a model and saves it to a path.

    Args:
        model_id (str): DOI of the model
        path (str): Destination to write model to.
        callback (func): Function called for every 1024 octet chunk received.

    Returns:
        The identifier the model can be called through on the command line.
        Will usually be the file name of the model.
    """
    logger.info(f'Saving model {model_id} to {path}')
    r = requests.get(f'{MODEL_REPO}records', params={'q': f'doi:"{model_id}"'})
    r.raise_for_status()
    callback(0, 0)
    resp = r.json()
    if resp['hits']['total'] != 1:
        logger.error(f'Found {resp["hits"]["total"]} models when querying for id \'{model_id}\'')
        raise KrakenRepoException(f'Found {resp["hits"]["total"]} models when querying for id \'{model_id}\'')

    metadata = resp['hits']['hits'][0]
    model_url = [x['links']['self'] for x in metadata['files'] if x['type'] == 'mlmodel'][0]
    # callable model identifier
    nat_id = os.path.basename(urllib.parse.urlparse(model_url).path)
    spath = os.path.join(path, nat_id)
    logger.debug(f'downloading model file {model_url} to {spath}')
    with closing(requests.get(model_url, stream=True)) as r:
        file_size = int(r.headers['Content-length'])
        with open(spath, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                callback(file_size, len(chunk))
                f.write(chunk)
    return nat_id

def get_description(model_id: str, callback: Callable[..., Any] = lambda: None) -> dict:
    """
    Fetches the metadata for a single model from the zenodo repository.

    Args:
        model_id (str): DOI of the model.
        callback (callable): Optional function called once per HTTP request.

    Returns:
        Dict
    """
    logger.info(f'Retrieving metadata for {model_id}')
    r = requests.get(f'{MODEL_REPO}records', params={'q': f'doi:"{model_id}"'})
    r.raise_for_status()
    callback()
    resp = r.json()
    if resp['hits']['total'] != 1:
        logger.error(f'Found {resp["hits"]["total"]} models when querying for id \'{model_id}\'')
        raise KrakenRepoException(f'Found {resp["hits"]["total"]} models when querying for id \'{model_id}\'')
    record = resp['hits']['hits'][0]
    metadata = record['metadata']
    if 'keywords' not in metadata:
        logger.error('No keywords included on deposit')
        raise KrakenRepoException('No keywords included on deposit.')
    model_type = SUPPORTED_MODELS.intersection(metadata['keywords'])
    if not model_type:
        msg = 'Unsupported model type(s): {}'.format(', '.join(metadata['keywords']))
        logger.error(msg)
        raise KrakenRepoException(msg)
    meta_json = None
    for file in record['files']:
        if file['key'] == 'metadata.json':
            callback()
            r = requests.get(file['links']['self'])
            r.raise_for_status()
            callback()
            try:
                meta_json = r.json()
            except Exception:
                msg = f'Metadata for \'{record["metadata"]["title"]}\' ({record["metadata"]["doi"]}) not in JSON format'
                logger.error(msg)
                raise KrakenRepoException(msg)
    if not meta_json:
        msg = 'Mo metadata.jsn found for \'{}\' ({})'.format(record['metadata']['title'], record['metadata']['doi'])
        logger.error(msg)
        raise KrakenRepoException(msg)
    # merge metadata.json into DataCite
    metadata.update({'graphemes': meta_json['graphemes'],
                     'summary': meta_json['summary'],
                     'script': meta_json['script'],
                     'link': record['links']['latest'],
                     'type': [x.split('_')[1] for x in model_type],
                     'accuracy': meta_json['accuracy']})
    return metadata

def get_listing(callback: Callable[[int, int], Any] = lambda total, advance: None) -> dict:
    """
    Fetches a listing of all kraken models from the zenodo repository.

    Args:
        callback (Callable): Function called after each HTTP request.

    Returns:
        Dict of models with each model.
    """
    logger.info('Retrieving model list')
    records = []
    r = requests.get('{}{}'.format(MODEL_REPO, 'records'), params={'communities': 'ocr_models'})
    r.raise_for_status()
    callback(1, 1)
    resp = r.json()
    if not resp['hits']['total']:
        logger.error('No models found in community \'ocr_models\'')
        raise KrakenRepoException('No models found in repository \'ocr_models\'')
    logger.debug('Total of {} records in repository'.format(resp['hits']['total']))
    total = resp['hits']['total']
    callback(total, 0)
    records.extend(resp['hits']['hits'])
    while 'next' in resp['links']:
        logger.debug('Fetching next page')
        r = requests.get(resp['links']['next'])
        r.raise_for_status()
        resp = r.json()
        logger.debug('Found {} new records'.format(len(resp['hits']['hits'])))
        records.extend(resp['hits']['hits'])
    logger.debug('Retrieving model metadata')
    models = {}
    # fetch metadata.jsn for each model
    for record in records:
        if 'keywords' not in record['metadata']:
            continue
        model_type = SUPPORTED_MODELS.intersection(record['metadata']['keywords'])
        if not model_type:
            continue
        for file in record['files']:
            if file['key'] == 'metadata.json':
                callback(total, 1)
                r = requests.get(file['links']['self'])
                r.raise_for_status()
                try:
                    metadata = r.json()
                except Exception:
                    msg = f'Metadata for \'{record["metadata"]["title"]}\' ({record["metadata"]["doi"]}) not in JSON format'
                    logger.error(msg)
                    raise KrakenRepoException(msg)
        # merge metadata.jsn into DataCite
        key = record['metadata']['doi']
        models[key] = record['metadata']
        models[key].update({'graphemes': metadata['graphemes'],
                            'summary': metadata['summary'],
                            'script': metadata['script'],
                            'link': record['links']['latest'],
                            'type': [x.split('_')[1] for x in model_type]})
    return models
<|endoftext|>


<commit_before># Copyright 2014 Ahmed H. Ismail

#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at

#        http://www.apache.org/licenses/LICENSE-2.0

#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
class Package(object):
    """Represents an analyzed Package.
    Fields:
        name : Mandatory, string.
        version: Optional, string.
        file_name: Optional, string.
        supplier: Optional, Organization or Person or NO_ASSERTION.
        originator: Optional, Organization or Person.
        download_location: Mandatory, URL as string.
        homepage: Optional, URL as string or None or NO_ASSERTION.
        verif_code: Mandatory string.
        check_sum: Optional string.
        source_info: Optional string.
        conc_lists: Mandatory string.
        licenses_from_files: License list, at least one.
        cr_text: Copyright text, string , NoAssertion or None. Mandatory.
        summary: Optional str.
        description: Optional str.
        files: List of files in package, atleast one.
    """
    def __init__(self, name=None, download_location=None,version="", file_name="",
            supplier=None, originator=None):
        super(Package, self).__init__()
        self.name = name
        self.version = version
        self.file_name = file_name
        self.supplier = supplier
        self.originator = originator
        self.download_location = download_location
        self.homepage = None
        self.verif_code = None
        self.check_sum = None
        self.source_info = None
        self.conc_lics = None
        self.license_declared = None
        self.license_comment = None
        self.licenses_from_files = []
        self.cr_text = None
        self.summary = None
        self.description = None
        self.files = []<commit_msg>Reformats package.py to conform to pep 8<commit_after># Copyright 2014 Ahmed H. Ismail

#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at

#        http://www.apache.org/licenses/LICENSE-2.0

#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

class Package(object):

    """Represents an analyzed Package.
    Fields:
        name : Mandatory, string.
        version: Optional, string.
        file_name: Optional, string.
        supplier: Optional, Organization or Person or NO_ASSERTION.
        originator: Optional, Organization or Person.
        download_location: Mandatory, URL as string.
        homepage: Optional, URL as string or None or NO_ASSERTION.
        verif_code: Mandatory string.
        check_sum: Optional string.
        source_info: Optional string.
        conc_lists: Mandatory string.
        licenses_from_files: License list, at least one.
        cr_text: Copyright text, string , NoAssertion or None. Mandatory.
        summary: Optional str.
        description: Optional str.
        files: List of files in package, atleast one.
    """

    def __init__(self, name=None, download_location=None, version="", file_name="",
                 supplier=None, originator=None):
        super(Package, self).__init__()
        self.name = name
        self.version = version
        self.file_name = file_name
        self.supplier = supplier
        self.originator = originator
        self.download_location = download_location
        self.homepage = None
        self.verif_code = None
        self.check_sum = None
        self.source_info = None
        self.conc_lics = None
        self.license_declared = None
        self.license_comment = None
        self.licenses_from_files = []
        self.cr_text = None
        self.summary = None
        self.description = None
        self.files = []
<|endoftext|>


<commit_before>import os
import shutil
import wget
import argparse
import sys
from tabulate import tabulate

from subprocess import call
from HTMLParser import HTMLParser

class Matrix(object):
    def __init__(self, group, name, id, rows, cols, nonZeros, file, valuetype, spd, sym, hasRhs):
       self.group = group
       self.name = name
       self.id = id
       self.rows = rows
       self.cols = cols
       self.nonZeros=nonZeros
       self.file=file
       self.valuetype=valuetype
       self.spd=spd
       self.sym=sym
       self.hasRhs=hasRhs

    def fullName(self):
        return self.group + '/' + self.name

    def __str__(self):
        return "group={} name={} id={} rows={} cols={} nonZeros={} file={} valuetype={} spd={} sym={} hasRhs={}".format(
               self.group, self.name, self.id, self.rows, self.cols,
               self.nonZeros, self.file, self.valuetype, self.spd, self.sym, self.hasRhs)

    def __repr__(self):
        return self.__str__()

class MatrixCollection(object):

  def __init__(self, matrixList):
    self.matrixList = matrixList

  def select(self, predicate):
    return MatrixCollection(filter(predicate, self.matrixList))

  def sorted(self, keyList, reverse=False):
    return MatrixCollection(
        sorted(self.matrixList,
          key=lambda x: tuple([getattr(x, f) for f in keyList]),
          reverse=not reverse))

  def download(self):
    for matrix in self.matrixList:
      url = 'http://www.cise.ufl.edu/research/sparse/MM/' + matrix.group + '/' + matrix.name + '.tar.gz'
      filename = wget.download(url)

  def each(self, function):
    pass

  def __str__(self):
    keys = ['group', 'name', 'id', 'rows', 'cols', 'nonZeros', 'file', 'valuetype', 'spd', 'sym', 'hasRhs']
    return tabulate(
        [[getattr(m, k) for k in keys] for m in self.matrixList],
        keys
    )

  def getSpdLinearSystems(self):
    return self.select(lambda x: x.hasRhs and x.sym =='yes' and x.spd == 'yes' )

  def __repr__(self):
    return self.__str__()

def ToInt(stringVal):
    return int(stringVal.replace(',', ''))

def fetchOtherProperties(matrices):
    with open('uof_rhs_names.txt') as f:
        names = set(map(lambda x : x.strip(), f.readlines()))
        for m in matrices:
            if m.fullName() in names:
                m.hasRhs = True;
    return matrices

class MyHtmlParser(HTMLParser):

    def __init__(self):
        HTMLParser.__init__(self)
        self.state = 'NONE'
        self.skipped_header = False
        self.value_fields = []
        self.matrices = []

    def handle_starttag(self, tag, attrs):
        if self.state == 'FINISHED':
            return
        if tag == 'table':
            self.state = 'PARSING_TABLE'
        elif tag == 'td':
            self.state ='PARSING_VALUE'
        elif tag == 'tr':
            if self.skipped_header:
                self.state = 'PARSING_ENTRY'
            else:
                self.skipped_header = True

    def handle_endtag(self, tag):
        if self.state == 'FINISHED':
            return
        if tag == 'table':
            self.state ='FINISHED'
        elif tag == 'td':
            self.state = 'PARSING_ENTRY'
        elif tag == 'tr':
            self.state = 'PARSING_TABLE'
            self.handle_matrix_entry()
            self.value_fields = []

    def handle_data(self, data):
        if self.state == 'FINISHED':
            return
        if self.state == 'PARSING_VALUE':
            data = data.strip()
            if "/,\n".find(data) == -1:
                self.value_fields.append(data)

    def handle_matrix_entry(self):
        fields = self.value_fields
        self.matrices.append(
            Matrix(
              group=fields[0],
              name=fields[1],
              id=fields[2],
              rows=ToInt(fields[6]),
              cols=ToInt(fields[7]),
              nonZeros=ToInt(fields[8]),
              file='',
              valuetype=fields[9].split()[0],
              sym=fields[10],
              spd=fields[11],
              hasRhs=None
            )
        )

def fetch(force=False):
    filename = 'list_by_nnz.html'
    if not os.path.isfile(filename) or force:
        print 'Fetching matrix list...'
        url = 'http://www.cise.ufl.edu/research/sparse/matrices/list_by_nnz.html'
        filename = wget.download(url)
        print 'Done'
    else:
        print '--> Using cached', filename, '; to force refetch, use the -f option'

    f = open(filename)
    parser = MyHtmlParser()
    parser.feed(f.read())
    return MatrixCollection(fetchOtherProperties(parser.matrices))

def main():

    parser = argparse.ArgumentParser(
        description='Download sparse matrices from UoF Collection.',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-n', '--dryrun',
                        action='store_true',
                        default=False,
                        help='Print matrices that would be downloaded.')
    parser.add_argument(
        '-c', '--containing',
        help='Only download matrices containing provided pattern')
    parser.add_argument(
        '-g', '--group',
        help='Only download matrices in a group containing provided pattern')
    parser.add_argument(
        '-a', '--all',
        help='Fetch all matrices matching default criteria')
    parser.add_argument(
        '-m', '--max-matrices',
        help='Maximum number of matrices to fetch',
        type=int,
        default=100)
    parser.add_argument(
        '-f', '--force',
        action='store_true',
        default=False,
        help='Force a refetch of the list_by_nnz.html file containing the index of all matrices')
    args = parser.parse_args()

    if len(sys.argv) == 1:
        parser.print_help()
        return

    if args.containing:
      print '--> Only fetching matrices containing \'', args.containing, '\''

    matrices = fetch(args.dryrun,
       args.containing,
       args.group,
       args.all,
       args.max_matrices,
       args.force).matrixList

    # print matrices
    print 'Fetched {} matrices'.format(len(matrices))

    # prepare matrices (move to directory, extract archive...)
    shutil.rmtree('matrices', True)
    os.mkdir('matrices')

    for matrix in matrices:
        if not matrix.file:
            continue
        print matrix.file
        shutil.move(matrix.file, 'matrices')
        call(['tar', '-xvzf', 'matrices/' + matrix.file, '-C', 'matrices/'])

if __name__ == '__main__':
    main()
<commit_msg>Save path to RHS, clean main<commit_after>import os
import shutil
import wget
import argparse
from tabulate import tabulate
from subprocess import call
from HTMLParser import HTMLParser

class Matrix(object):
    def __init__(self, group, name, id, rows, cols, nonZeros, file, valuetype, spd, sym, hasRhs):
       self.group = group
       self.name = name
       self.id = id
       self.rows = rows
       self.cols = cols
       self.nonZeros=nonZeros
       self.file=file
       self.valuetype=valuetype
       self.spd=spd
       self.sym=sym
       self.hasRhs=hasRhs
       self.rhsFile = None

    def fullName(self):
        return self.group + '/' + self.name

    def __str__(self):
        return "group={} name={} id={} rows={} cols={} nonZeros={} file={} valuetype={} spd={} sym={} hasRhs={} rhsFile={}".format(
               self.group, self.name, self.id, self.rows, self.cols,
               self.nonZeros, self.file, self.valuetype, self.spd, self.sym, self.hasRhs, self.rhsFile)

    def __repr__(self):
        return self.__str__()

class MatrixCollection(object):

  def __init__(self, matrixList):
    self.matrixList = matrixList

  def select(self, predicate):
    return MatrixCollection(filter(predicate, self.matrixList))

  def sorted(self, keyList, reverse=False):
    return MatrixCollection(
        sorted(self.matrixList,
          key=lambda x: tuple([getattr(x, f) for f in keyList]),
          reverse=not reverse))

  def download(self, dir='matrices'):
    shutil.rmtree(dir, True)
    os.mkdir(dir)
    for m in self.matrixList:
      print 'Fetching {}'.format(m.fullName())
      url = 'http://www.cise.ufl.edu/research/sparse/MM/' + m.fullName() + '.tar.gz'
      m.file = wget.download(url)
      print
      shutil.move(m.file, dir)
      print 'Extracting...'
      call(['tar', '-xvzf', os.path.join(dir, m.file), '-C', dir])
      m.file = os.path.abspath(os.path.join(dir, m.name, m.name + '.mtx'))
      if not os.path.exists(m.file):
          print 'Warning! unexpected name for matrix', m.name()
      if m.hasRhs:
          m.rhsFile = os.path.abspath(os.path.join(dir, m.name, m.name + '_b.mtx'))
          if not os.path.exists(m.rhsFile):
              print 'Warning! unexpected name for RHS of system', m.name()
      print m
      print

  def each(self, function):
    pass

  def __str__(self):
    keys = ['group', 'name', 'id', 'rows', 'cols', 'nonZeros', 'valuetype', 'spd', 'sym', 'hasRhs']
    return tabulate([[getattr(m, k) for k in keys] for m in self.matrixList], keys)

  def getSpdLinearSystems(self):
    return self.select(lambda x: x.hasRhs and x.sym =='yes' and x.spd == 'yes' )

  def __repr__(self):
    return self.__str__()

def ToInt(stringVal):
    return int(stringVal.replace(',', ''))

def fetchOtherProperties(matrices):
    with open('uof_rhs_names.txt') as f:
        names = set(map(lambda x : x.strip(), f.readlines()))
        for m in matrices:
            if m.fullName() in names:
                m.hasRhs = True;
    return matrices

class MyHtmlParser(HTMLParser):

    def __init__(self):
        HTMLParser.__init__(self)
        self.state = 'NONE'
        self.skipped_header = False
        self.value_fields = []
        self.matrices = []

    def handle_starttag(self, tag, attrs):
        if self.state == 'FINISHED':
            return
        if tag == 'table':
            self.state = 'PARSING_TABLE'
        elif tag == 'td':
            self.state ='PARSING_VALUE'
        elif tag == 'tr':
            if self.skipped_header:
                self.state = 'PARSING_ENTRY'
            else:
                self.skipped_header = True

    def handle_endtag(self, tag):
        if self.state == 'FINISHED':
            return
        if tag == 'table':
            self.state ='FINISHED'
        elif tag == 'td':
            self.state = 'PARSING_ENTRY'
        elif tag == 'tr':
            self.state = 'PARSING_TABLE'
            self.handle_matrix_entry()
            self.value_fields = []

    def handle_data(self, data):
        if self.state == 'FINISHED':
            return
        if self.state == 'PARSING_VALUE':
            data = data.strip()
            if "/,\n".find(data) == -1:
                self.value_fields.append(data)

    def handle_matrix_entry(self):
        fields = self.value_fields
        self.matrices.append(
            Matrix(
              group=fields[0],
              name=fields[1],
              id=fields[2],
              rows=ToInt(fields[6]),
              cols=ToInt(fields[7]),
              nonZeros=ToInt(fields[8]),
              file='',
              valuetype=fields[9].split()[0],
              sym=fields[10],
              spd=fields[11],
              hasRhs=None
            )
        )

def fetch(force=False):
    filename = 'list_by_nnz.html'
    if not os.path.isfile(filename) or force:
        print 'Fetching matrix list...'
        url = 'http://www.cise.ufl.edu/research/sparse/matrices/list_by_nnz.html'
        filename = wget.download(url)
        print 'Done'
    else:
        print '--> Using cached', filename, '; to force refetch, use the -f option'

    f = open(filename)
    parser = MyHtmlParser()
    parser.feed(f.read())
    return MatrixCollection(fetchOtherProperties(parser.matrices))

def main():

    parser = argparse.ArgumentParser(
        description='Download sparse matrices from UoF Collection.',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('-n', '--dryrun',
                        action='store_true',
                        default=False,
                        help='Print matrices that would be downloaded.')
    parser.add_argument(
        '-c', '--containing',
        help='Only download matrices containing provided pattern')
    parser.add_argument(
        '-g', '--group',
        help='Only download matrices in a group containing provided pattern')
    parser.add_argument(
        '-m', '--max-matrices',
        help='Maximum number of matrices to fetch',
        type=int,
        default=100)
    parser.add_argument(
        '-f', '--force',
        action='store_true',
        default=False,
        help='Force a refetch of the list_by_nnz.html file containing the index of all matrices')
    args = parser.parse_args()

    matrices = fetch(args.force).select(
        lambda x : (not args.containing or args.containing in x.name) and (not args.group or args.group in x.group))
    matrices = MatrixCollection(matrices.matrixList[:args.max_matrices])

    if args.dryrun:
        print matrices
        print 'Fetched {} matrices'.format(len(matrices))
        return

    matrices.download()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import datetime

class Maintenance(object):

    __source = None
    __instance = None

    def __init__(self, source, instance):
        self.__source = source
        self.__instance = instance

    @property
    def is_enabled(self):
        return self.__source['maintenance']

    @property
    def reason(self):
        return self.__source.get('reason', None)

    @property
    def user(self):
        return self.__source.get('username', None)

    @property
    def expires(self):
        try:
            timestamp = self.__source['schedule']['expires_epoch']
            return datetime.datetime.fromtimestamp(timestamp)
        except KeyError:
            return None

    def enable(self, reason, expires=None):
        body = {
            'username': self.__instance._Instance__client.authorization[0],
            'reason': reason,
            'maintenance': True
        }

        if expires:
            delta = expires - datetime.datetime(1970, 1, 1)
            expires_epoch = int(delta.total_seconds())

            body['schedule'] = {
                'expires_epoch': expires_epoch
            }

        endpoint = 'instances/{id}/maintenance/'.format(id=self.__instance.id)

        response = self.__instance._Instance__client.request(method='PUT',
                                                             endpoint=endpoint,
                                                             body=body)

        return (response.status_code == 200)

class Instance(object):

    __source = None
    __client = None

    def __init__(self, source, client):
        self.__source = source
        self.__client = client

    @property
    def id(self):
        return self.__source['id']

    @property
    def instance_id(self):
        return self.__source['instance_id']

    @property
    def name(self):
        return self.__source['name']

    @property
    def provider(self):
        return self.__source['provider']

    @property
    def provider_region(self):
        return self.__source['provider_region']

    @property
    def provider_zone(self):
        return self.__source['provider_zone']

    @property
    def provider_account(self):
        return self.__source['provider_account']

    @property
    def instance_type(self):
        return self.__source['instance_type']

    @property
    def interfaces(self):
        return self.__source['interfaces']

    @property
    def image(self):
        return self.__source['image']

    @property
    def tags(self):
        return {tag['name']: tag['value'] for tag in self.__source['tags']}

    @property
    def launched(self):
        timestamp = self.__source['launch_epoch']

        return datetime.datetime.fromtimestamp(timestamp)

    @property
    def last_monitored(self):
        timestamp = self.__source['monitor_epoch']

        return datetime.datetime.fromtimestamp(timestamp)

    @property
    def state(self):
        return self.__source['state']

    @property
    def agent_version(self):
        return self.__source['agent_version']

    @property
    def extractor_version(self):
        return self.__source['extractor_version']

    @property
    def maintenance(self):
        try:
            return self.__source['maintenance_mode']
        except KeyError:
            endpoint = 'instances/{id}/maintenance/'.format(id=self.id)
            response = self.__client.request(endpoint=endpoint).json()

            self.__source['maintenance_mode'] = Maintenance(response['data'],
                                                            self)

            return self.__source['maintenance_mode']
<commit_msg>Added support for disabling maintenance mode<commit_after>import datetime

class Maintenance(object):

    __source = None
    __instance = None

    def __init__(self, source, instance):
        self.__source = source
        self.__instance = instance

    @property
    def is_enabled(self):
        return self.__source['maintenance']

    @property
    def reason(self):
        return self.__source.get('reason', None)

    @property
    def user(self):
        return self.__source.get('username', None)

    @property
    def expires(self):
        try:
            timestamp = self.__source['schedule']['expires_epoch']
            return datetime.datetime.fromtimestamp(timestamp)
        except KeyError:
            return None

    def enable(self, reason, expires=None):
        body = {
            'username': self.__instance._Instance__client.authorization[0],
            'reason': reason,
            'maintenance': True
        }

        if expires:
            delta = expires - datetime.datetime(1970, 1, 1)
            expires_epoch = int(delta.total_seconds())

            body['schedule'] = {
                'expires_epoch': expires_epoch
            }

        endpoint = 'instances/{id}/maintenance/'.format(id=self.__instance.id)

        response = self.__instance._Instance__client.request(method='PUT',
                                                             endpoint=endpoint,
                                                             body=body)

        return (response.status_code == 200)

    def disable(self, reason, expires=None):
        body = {
            'username': self.__instance._Instance__client.authorization[0],
            'reason': reason,
            'maintenance': False
        }

        if expires:
            delta = expires - datetime.datetime(1970, 1, 1)
            expires_epoch = int(delta.total_seconds())

            body['schedule'] = {
                'expires_epoch': expires_epoch
            }

        endpoint = 'instances/{id}/maintenance/'.format(id=self.__instance.id)

        response = self.__instance._Instance__client.request(method='PUT',
                                                             endpoint=endpoint,
                                                             body=body)

        return (response.status_code == 200)

class Instance(object):

    __source = None
    __client = None

    def __init__(self, source, client):
        self.__source = source
        self.__client = client

    @property
    def id(self):
        return self.__source['id']

    @property
    def instance_id(self):
        return self.__source['instance_id']

    @property
    def name(self):
        return self.__source['name']

    @property
    def provider(self):
        return self.__source['provider']

    @property
    def provider_region(self):
        return self.__source['provider_region']

    @property
    def provider_zone(self):
        return self.__source['provider_zone']

    @property
    def provider_account(self):
        return self.__source['provider_account']

    @property
    def instance_type(self):
        return self.__source['instance_type']

    @property
    def interfaces(self):
        return self.__source['interfaces']

    @property
    def image(self):
        return self.__source['image']

    @property
    def tags(self):
        return {tag['name']: tag['value'] for tag in self.__source['tags']}

    @property
    def launched(self):
        timestamp = self.__source['launch_epoch']

        return datetime.datetime.fromtimestamp(timestamp)

    @property
    def last_monitored(self):
        timestamp = self.__source['monitor_epoch']

        return datetime.datetime.fromtimestamp(timestamp)

    @property
    def state(self):
        return self.__source['state']

    @property
    def agent_version(self):
        return self.__source['agent_version']

    @property
    def extractor_version(self):
        return self.__source['extractor_version']

    @property
    def maintenance(self):
        try:
            return self.__source['maintenance_mode']
        except KeyError:
            endpoint = 'instances/{id}/maintenance/'.format(id=self.id)
            response = self.__client.request(endpoint=endpoint).json()

            self.__source['maintenance_mode'] = Maintenance(response['data'],
                                                            self)

            return self.__source['maintenance_mode']
<|endoftext|>


<commit_before><commit_msg>FEM: ccx tools test, sort imports<commit_after><|endoftext|>


<commit_before>#! /usr/bin/env python
#
# Dakota utility programs.
#
# Mark Piper (mark.piper@colorado.edu)

import numpy as np

def get_names(dat_file):
    '''
    Reads the header from Dakota tabular graphics file. Returns a list
    of variable names.
    '''
    fp = open(dat_file, 'r')
    return fp.readline().split()

def get_data(dat_file):
    '''
    Reads the data from Dakota tabular graphics file. Returns a numpy array.
    '''
    return np.loadtxt(dat_file, skiprows=1, unpack=True)

def read_tabular(dat_file):
    '''
    Reads a Dakota tabular graphics file. Returns a dict with variable
    names and a numpy array with data.
    '''
    names = get_names(dat_file)
    data = get_data(dat_file)
    return {'names':names, 'data':data}
<commit_msg>Add routine to make surface plot from Dakota output<commit_after>#! /usr/bin/env python
#
# Dakota utility programs.
#
# Mark Piper (mark.piper@colorado.edu)

import numpy as np
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt

def get_names(dat_file):
    '''
    Reads the header from Dakota tabular graphics file. Returns a list
    of variable names.
    '''
    fp = open(dat_file, 'r')
    return fp.readline().split()

def get_data(dat_file):
    '''
    Reads the data from Dakota tabular graphics file. Returns a numpy array.
    '''
    return np.loadtxt(dat_file, skiprows=1, unpack=True)

def read_tabular(dat_file):
    '''
    Reads a Dakota tabular graphics file. Returns a dict with variable
    names and a numpy array with data.
    '''
    names = get_names(dat_file)
    data = get_data(dat_file)
    return {'names':names, 'data':data}

def plot_tabular_2d(tab_data, column_index=-1):
    '''
    Surface plot.
    '''
    x = tab_data.get('data')[1,]
    y = tab_data.get('data')[2,]
    z = tab_data.get('data')[column_index,]

    m = len(set(x))
    n = len(set(y))

    X = x.reshape(m,n)
    Y = y.reshape(m,n)
    Z = z.reshape(m,n)

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_surface(X, Y, Z, rstride=1, cstride=1)
    ax.set_xlabel(tab_data.get('names')[1])
    ax.set_ylabel(tab_data.get('names')[2])
    ax.set_zlabel(tab_data.get('names')[column_index])
    plt.show(block=False)
<|endoftext|>


<commit_before># Copyright (c) 2014 Ahmed H. Ismail
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

import hashlib

from six.moves import reduce

from spdx import checksum
from spdx import creationinfo
from spdx import document
from spdx import utils

class Package(object):

    """
    Represent an analyzed Package.
    Fields:
     - name : Mandatory, string.
     - spdx_id: Uniquely identify any element in an SPDX document which may be
     referenced by other elements. Mandatory, one. Type: str.
     - version: Optional, string.
     - file_name: Optional, string.
     - supplier: Optional, Organization or Person or NO_ASSERTION.
     - originator: Optional, Organization or Person.
     - download_location: Mandatory, URL as string.
     - files_analyzed: Indicates whether the file content of this package has
     been available for or subjected to analysis when creating the SPDX
     document. If "false" indicates packages that represent metadata or URI
     references to a project, product, artifact, distribution or a component.
     If set to "false", the package must not contain any files.
     Optional, boolean.
     - homepage: Optional, URL as string or NONE or NO_ASSERTION.
     - verif_code: Mandatory string.
     - check_sum: Optional , spdx.checksum.Algorithm.
     - source_info: Optional string.
     - conc_lics: Mandatory spdx.document.License or spdx.utils.SPDXNone or
     - spdx.utils.NoAssert.
     - license_declared : Mandatory spdx.document.License or spdx.utils.SPDXNone or
     - spdx.utils.NoAssert.
     - license_comment  : optional string.
     - licenses_from_files: list of spdx.document.License or spdx.utils.SPDXNone or
     - spdx.utils.NoAssert.
     - cr_text: Copyright text, string , utils.NoAssert or utils.SPDXNone. Mandatory.
     - summary: Optional str.
     - description: Optional str.
     - comment: Comments about the package being described, optional one.
     Type: str
     - files: List of files in package, atleast one.
     - verif_exc_files : list of file names excluded from verification code or None.
     - ext_pkg_refs : External references referenced within the given package.
     Optional, one or many. Type: ExternalPackageRef
     - attribution_text : optional string.
    """

    def __init__(
        self,
        name=None,
        spdx_id=None,
        download_location=None,
        version=None,
        file_name=None,
        supplier=None,
        originator=None,
    ):
        self.name = name
        self.spdx_id = spdx_id
        self.version = version
        self.file_name = file_name
        self.supplier = supplier
        self.originator = originator
        self.download_location = download_location
        self.files_analyzed = None
        self.homepage = None
        self.verif_code = None
        self.check_sum = None
        self.source_info = None
        self.conc_lics = None
        self.license_declared = None
        self.license_comment = None
        self.licenses_from_files = []
        self.cr_text = None
        self.summary = None
        self.description = None
        self.comment = None
        self.attribution_text = None
        self.files = []
        self.verif_exc_files = []
        self.pkg_ext_refs = []

    def add_file(self, fil):
        self.files.append(fil)

    def add_lics_from_file(self, lics):
        self.licenses_from_files.append(lics)

    def add_exc_file(self, filename):
        self.verif_exc_files.append(filename)

    def add_pkg_ext_refs(self, pkg_ext_ref):
        self.pkg_ext_refs.append(pkg_ext_ref)

    def validate(self, messages=None):
        """
        Validate the package fields.
        Append user friendly error messages to the `messages` list.
        """
        messages = self.validate_checksum(messages)
        messages = self.validate_optional_str_fields(messages)
        messages = self.validate_mandatory_str_fields(messages)
        messages = self.validate_files(messages)
        messages = self.validate_pkg_ext_refs(messages)
        messages = self.validate_mandatory_fields(messages)
        messages = self.validate_optional_fields(messages)

        return messages

    def validate_optional_fields(self, messages):
        if self.originator and not isinstance(
            self.originator, (utils.NoAssert, creationinfo.Creator)
        ):
            messages = messages + [
                "Package originator must be instance of "
                "spdx.utils.NoAssert or spdx.creationinfo.Creator"
            ]

        if self.supplier and not isinstance(
            self.supplier, (utils.NoAssert, creationinfo.Creator)
        ):
            messages = messages + [
                "Package supplier must be instance of "
                "spdx.utils.NoAssert or spdx.creationinfo.Creator"
            ]

        return messages

    def validate_pkg_ext_refs(self, messages=None):
        for ref in self.pkg_ext_refs:
            if isinstance(ref, ExternalPackageRef):
                messages = ref.validate(messages)
            else:
                messages = messages + [
                    "External package references must be of the type "
                    "spdx.package.ExternalPackageRef and not " + str(type(ref))
                ]

        return messages

    def validate_mandatory_fields(self, messages):
        if not isinstance(
            self.conc_lics, (utils.SPDXNone, utils.NoAssert, document.License)
        ):
            messages = messages + [
                "Package concluded license must be instance of "
                "spdx.utils.SPDXNone or spdx.utils.NoAssert or "
                "spdx.document.License"
            ]

        if not isinstance(
            self.license_declared, (utils.SPDXNone, utils.NoAssert, document.License)
        ):
            messages = messages + [
                "Package declared license must be instance of "
                "spdx.utils.SPDXNone or spdx.utils.NoAssert or "
                "spdx.document.License"
            ]

        # FIXME: this is obscure and unreadable
        license_from_file_check = lambda prev, el: prev and isinstance(
            el, (document.License, utils.SPDXNone, utils.NoAssert)
        )
        if not reduce(license_from_file_check, self.licenses_from_files, True):
            messages = messages + [
                "Each element in licenses_from_files must be instance of "
                "spdx.utils.SPDXNone or spdx.utils.NoAssert or "
                "spdx.document.License"
            ]

        if not self.licenses_from_files:
            messages = messages + ["Package licenses_from_files can not be empty"]

        return messages

    def validate_files(self, messages):
        if not self.files:
            messages = messages + ["Package must have at least one file."]
        else:
            for f in self.files:
                messages = f.validate(messages)

        return messages

    def validate_optional_str_fields(self, messages):
        """Fields marked as optional and of type string in class
        docstring must be of a type that provides __str__ method.
        """
        FIELDS = [
            "file_name",
            "version",
            "homepage",
            "source_info",
            "summary",
            "description",
            "attribution_text",
            "comment",
        ]
        messages = self.validate_str_fields(FIELDS, True, messages)

        return messages

    def validate_mandatory_str_fields(self, messages):
        """Fields marked as Mandatory and of type string in class
        docstring must be of a type that provides __str__ method.
        """
        FIELDS = ["name", "spdx_id", "download_location", "verif_code", "cr_text"]
        messages = self.validate_str_fields(FIELDS, False, messages)

        return messages

    def validate_str_fields(self, fields, optional, messages):
        """Helper for validate_mandatory_str_field and
        validate_optional_str_fields"""
        for field_str in fields:
            field = getattr(self, field_str)
            if field is not None:
                # FIXME: this does not make sense???
                attr = getattr(field, "__str__", None)
                if not callable(attr):
                    messages = messages + [
                        "{0} must provide __str__ method.".format(field)
                    ]
                    # Continue checking.
            elif not optional:
                messages = messages + ["Package {0} can not be None.".format(field_str)]

        return messages

    def validate_checksum(self, messages):
        if not isinstance(self.check_sum, checksum.Algorithm):
            messages = messages + [
                "Package checksum must be instance of spdx.checksum.Algorithm"
            ]
        else:
            if self.check_sum.identifier != "SHA1":
                messages = messages + ["File checksum algorithm must be SHA1"]

        return messages

    def calc_verif_code(self):
        hashes = []

        for file_entry in self.files:
            if (
                isinstance(file_entry.chk_sum, checksum.Algorithm)
                and file_entry.chk_sum.identifier == "SHA1"
            ):
                sha1 = file_entry.chk_sum.value
            else:
                sha1 = file_entry.calc_chksum()
            hashes.append(sha1)

        hashes.sort()

        sha1 = hashlib.sha1()
        sha1.update("".join(hashes).encode("utf-8"))
        return sha1.hexdigest()

    def has_optional_field(self, field):
        return getattr(self, field, None) is not None

class ExternalPackageRef(object):
    """
    An External Reference allows a Package to reference an external source of
    additional information, metadata, enumerations, asset identifiers, or
    downloadable content believed to be relevant to the Package.
    Fields:
    - category: "SECURITY" or "PACKAGE-MANAGER" or "OTHER".
    - pkg_ext_ref_type: A unique string containing letters, numbers, ".","-".
    - locator: A unique string with no spaces necessary to access the
    package-specific information, metadata, or content within the target
    location.
    - comment: To provide information about the purpose and target of the
    reference.
    """

    def __init__(
        self, category=None, pkg_ext_ref_type=None, locator=None, comment=None
    ):
        self.category = category
        self.pkg_ext_ref_type = pkg_ext_ref_type
        self.locator = locator
        self.comment = comment

    def validate(self, messages=None):
        """
        Validate all fields of the ExternalPackageRef class and update the
        messages list with user friendly error messages for display.
        """
        messages = self.validate_category(messages)
        messages = self.validate_pkg_ext_ref_type(messages)
        messages = self.validate_locator(messages)

        return messages

    def validate_category(self, messages=None):
        if self.category is None:
            messages = messages + ["ExternalPackageRef has no category."]

        return messages

    def validate_pkg_ext_ref_type(self, messages=None):
        if self.pkg_ext_ref_type is None:
            messages = messages + ["ExternalPackageRef has no type."]

        return messages

    def validate_locator(self, messages=None):
        if self.locator is None:
            messages = messages + ["ExternalPackageRef has no locator."]

        return messages
<commit_msg>fix validation errors when files_analyzed is False - solve issue #149<commit_after># Copyright (c) 2014 Ahmed H. Ismail
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import print_function
from __future__ import unicode_literals

import hashlib

from six.moves import reduce

from spdx import checksum
from spdx import creationinfo
from spdx import document
from spdx import utils

class Package(object):

    """
    Represent an analyzed Package.
    Fields:
     - name : Mandatory, string.
     - spdx_id: Uniquely identify any element in an SPDX document which may be
     referenced by other elements. Mandatory, one. Type: str.
     - version: Optional, string.
     - file_name: Optional, string.
     - supplier: Optional, Organization or Person or NO_ASSERTION.
     - originator: Optional, Organization or Person.
     - download_location: Mandatory, URL as string.
     - files_analyzed: Indicates whether the file content of this package has
     been available for or subjected to analysis when creating the SPDX
     document. If "false" indicates packages that represent metadata or URI
     references to a project, product, artifact, distribution or a component.
     If set to "false", the package must not contain any files.
     Optional, boolean.
     - homepage: Optional, URL as string or NONE or NO_ASSERTION.
     - verif_code: string. Mandatory if files_analyzed is True or None (omitted)
       Must be None (omitted) if files_analyzed is False
     - check_sum: Optional , spdx.checksum.Algorithm.
     - source_info: Optional string.
     - conc_lics: Mandatory spdx.document.License or spdx.utils.SPDXNone or
     - spdx.utils.NoAssert.
     - license_declared : Mandatory spdx.document.License or spdx.utils.SPDXNone or
     - spdx.utils.NoAssert.
     - license_comment  : optional string.
     - licenses_from_files: list of spdx.document.License or spdx.utils.SPDXNone or
     - spdx.utils.NoAssert.
     - cr_text: Copyright text, string , utils.NoAssert or utils.SPDXNone. Mandatory.
     - summary: Optional str.
     - description: Optional str.
     - comment: Comments about the package being described, optional one.
     Type: str
     - files: List of files in package, atleast one.
     - verif_exc_files : list of file names excluded from verification code or None.
     - ext_pkg_refs : External references referenced within the given package.
     Optional, one or many. Type: ExternalPackageRef
     - attribution_text : optional string.
    """

    def __init__(
        self,
        name=None,
        spdx_id=None,
        download_location=None,
        version=None,
        file_name=None,
        supplier=None,
        originator=None,
    ):
        self.name = name
        self.spdx_id = spdx_id
        self.version = version
        self.file_name = file_name
        self.supplier = supplier
        self.originator = originator
        self.download_location = download_location
        self.files_analyzed = None
        self.homepage = None
        self.verif_code = None
        self.check_sum = None
        self.source_info = None
        self.conc_lics = None
        self.license_declared = None
        self.license_comment = None
        self.licenses_from_files = []
        self.cr_text = None
        self.summary = None
        self.description = None
        self.comment = None
        self.attribution_text = None
        self.files = []
        self.verif_exc_files = []
        self.pkg_ext_refs = []

    def add_file(self, fil):
        self.files.append(fil)

    def add_lics_from_file(self, lics):
        self.licenses_from_files.append(lics)

    def add_exc_file(self, filename):
        self.verif_exc_files.append(filename)

    def add_pkg_ext_refs(self, pkg_ext_ref):
        self.pkg_ext_refs.append(pkg_ext_ref)

    def validate(self, messages=None):
        """
        Validate the package fields.
        Append user friendly error messages to the `messages` list.
        """
        messages = self.validate_files_analyzed(messages)
        messages = self.validate_checksum(messages)
        messages = self.validate_optional_str_fields(messages)
        messages = self.validate_mandatory_str_fields(messages)
        messages = self.validate_files(messages)
        messages = self.validate_pkg_ext_refs(messages)
        messages = self.validate_mandatory_fields(messages)
        messages = self.validate_optional_fields(messages)

        return messages

    def validate_files_analyzed(self, messages):
        if self.files_analyzed not in [ True, False, None ]:
            messages = messages + [
                'Package files_analyzed must be True or False or None (omitted)'
            ]
        if self.files_analyzed == False and self.verif_code is not None:
            messages = messages + [
                'Package verif_code must be None (omitted) when files_analyzed is False'
            ]

        return messages

    def validate_optional_fields(self, messages):
        if self.originator and not isinstance(
            self.originator, (utils.NoAssert, creationinfo.Creator)
        ):
            messages = messages + [
                "Package originator must be instance of "
                "spdx.utils.NoAssert or spdx.creationinfo.Creator"
            ]

        if self.supplier and not isinstance(
            self.supplier, (utils.NoAssert, creationinfo.Creator)
        ):
            messages = messages + [
                "Package supplier must be instance of "
                "spdx.utils.NoAssert or spdx.creationinfo.Creator"
            ]

        return messages

    def validate_pkg_ext_refs(self, messages=None):
        for ref in self.pkg_ext_refs:
            if isinstance(ref, ExternalPackageRef):
                messages = ref.validate(messages)
            else:
                messages = messages + [
                    "External package references must be of the type "
                    "spdx.package.ExternalPackageRef and not " + str(type(ref))
                ]

        return messages

    def validate_mandatory_fields(self, messages):
        if not isinstance(
            self.conc_lics, (utils.SPDXNone, utils.NoAssert, document.License)
        ):
            messages = messages + [
                "Package concluded license must be instance of "
                "spdx.utils.SPDXNone or spdx.utils.NoAssert or "
                "spdx.document.License"
            ]

        if not isinstance(
            self.license_declared, (utils.SPDXNone, utils.NoAssert, document.License)
        ):
            messages = messages + [
                "Package declared license must be instance of "
                "spdx.utils.SPDXNone or spdx.utils.NoAssert or "
                "spdx.document.License"
            ]

        # FIXME: this is obscure and unreadable
        license_from_file_check = lambda prev, el: prev and isinstance(
            el, (document.License, utils.SPDXNone, utils.NoAssert)
        )
        if not reduce(license_from_file_check, self.licenses_from_files, True):
            messages = messages + [
                "Each element in licenses_from_files must be instance of "
                "spdx.utils.SPDXNone or spdx.utils.NoAssert or "
                "spdx.document.License"
            ]

        if not self.licenses_from_files:
            messages = messages + ["Package licenses_from_files can not be empty"]

        return messages

    def validate_files(self, messages):
        if self.files_analyzed != False:
            if not self.files:
                messages = messages + [
                    "Package must have at least one file."
                ]
            else:
                for f in self.files:
                    messages = f.validate(messages)

        return messages

    def validate_optional_str_fields(self, messages):
        """Fields marked as optional and of type string in class
        docstring must be of a type that provides __str__ method.
        """
        FIELDS = [
            "file_name",
            "version",
            "homepage",
            "source_info",
            "summary",
            "description",
            "attribution_text",
            "comment",
        ]
        messages = self.validate_str_fields(FIELDS, True, messages)

        return messages

    def validate_mandatory_str_fields(self, messages):
        """Fields marked as Mandatory and of type string in class
        docstring must be of a type that provides __str__ method.
        """
        FIELDS = ["name", "spdx_id", "download_location", "cr_text"]
        if self.files_analyzed != False:
            FIELDS = FIELDS + ["verif_code"]
        messages = self.validate_str_fields(FIELDS, False, messages)

        return messages

    def validate_str_fields(self, fields, optional, messages):
        """Helper for validate_mandatory_str_field and
        validate_optional_str_fields"""
        for field_str in fields:
            field = getattr(self, field_str)
            if field is not None:
                # FIXME: this does not make sense???
                attr = getattr(field, "__str__", None)
                if not callable(attr):
                    messages = messages + [
                        "{0} must provide __str__ method.".format(field)
                    ]
                    # Continue checking.
            elif not optional:
                messages = messages + ["Package {0} can not be None.".format(field_str)]

        return messages

    def validate_checksum(self, messages):
        if self.check_sum is not None: # optional
            if not isinstance(self.check_sum, checksum.Algorithm):
                messages = messages + [
                    "Package checksum must be instance of spdx.checksum.Algorithm"
                ]
            else:
                if self.check_sum.identifier != "SHA1":
                    messages = messages + ["File checksum algorithm must be SHA1"]

        return messages

    def calc_verif_code(self):
        hashes = []

        for file_entry in self.files:
            if (
                isinstance(file_entry.chk_sum, checksum.Algorithm)
                and file_entry.chk_sum.identifier == "SHA1"
            ):
                sha1 = file_entry.chk_sum.value
            else:
                sha1 = file_entry.calc_chksum()
            hashes.append(sha1)

        hashes.sort()

        sha1 = hashlib.sha1()
        sha1.update("".join(hashes).encode("utf-8"))
        return sha1.hexdigest()

    def has_optional_field(self, field):
        return getattr(self, field, None) is not None

class ExternalPackageRef(object):
    """
    An External Reference allows a Package to reference an external source of
    additional information, metadata, enumerations, asset identifiers, or
    downloadable content believed to be relevant to the Package.
    Fields:
    - category: "SECURITY" or "PACKAGE-MANAGER" or "OTHER".
    - pkg_ext_ref_type: A unique string containing letters, numbers, ".","-".
    - locator: A unique string with no spaces necessary to access the
    package-specific information, metadata, or content within the target
    location.
    - comment: To provide information about the purpose and target of the
    reference.
    """

    def __init__(
        self, category=None, pkg_ext_ref_type=None, locator=None, comment=None
    ):
        self.category = category
        self.pkg_ext_ref_type = pkg_ext_ref_type
        self.locator = locator
        self.comment = comment

    def validate(self, messages=None):
        """
        Validate all fields of the ExternalPackageRef class and update the
        messages list with user friendly error messages for display.
        """
        messages = self.validate_category(messages)
        messages = self.validate_pkg_ext_ref_type(messages)
        messages = self.validate_locator(messages)

        return messages

    def validate_category(self, messages=None):
        if self.category is None:
            messages = messages + ["ExternalPackageRef has no category."]

        return messages

    def validate_pkg_ext_ref_type(self, messages=None):
        if self.pkg_ext_ref_type is None:
            messages = messages + ["ExternalPackageRef has no type."]

        return messages

    def validate_locator(self, messages=None):
        if self.locator is None:
            messages = messages + ["ExternalPackageRef has no locator."]

        return messages
<|endoftext|>


<commit_before>import math
from nc.nc import *
import area

def make_smaller( curve, start = None, finish = None ):
    if start != None:
        curve.ChangeStart(start)

    if finish != None:
        curve.ChangeEnd(finish)

class Tag:
    def __init__(self, p, width, angle, height):
        self.p = p
        self.width = width # measured at the top of the tag. In the toolpath, the tag width will be this with plus the tool diameter, so that the finished tag has this "width" at it's smallest
        self.angle = angle # the angle of the ramp in radians. Between 0 and Pi/2; 0 is horizontal, Pi/2 is vertical
        self.height = height # the height of the tag, always measured above "final_depth"
        self.ramp_width = self.height / math.tan(self.angle)

    def split_curve(self, curve, radius, start_depth, depth, final_depth):
        tag_top_depth = final_depth + self.height

        if depth > tag_top_depth - 0.0000001:
            return # kurve is above this tag, so doesn't need splitting

        height_above_depth = tag_top_depth - depth
        ramp_width_at_depth = height_above_depth / math.tan(self.angle)
        cut_depth = start_depth - depth
        half_flat_top = radius + self.width / 2

        d = curve.PointToPerim(self.p)
        d0 = d - half_flat_top
        perim = curve.Perim()
        if curve.IsClosed():
            while d0 < 0: d0 += perim
            while d0 > perim: d0 -= perim
        p = curve.PerimToPoint(d0)
        curve.Break(p)
        d1 = d + half_flat_top
        if curve.IsClosed():
            while d1 < 0: d1 += perim
            while d1 > perim: d1 -= perim
        p = curve.PerimToPoint(d1)
        curve.Break(p)

        d0 = d - half_flat_top - ramp_width_at_depth
        if curve.IsClosed():
            while d0 < 0: d0 += perim
            while d0 > perim: d0 -= perim
        p = curve.PerimToPoint(d0)
        curve.Break(p)
        d1 = d + half_flat_top + ramp_width_at_depth
        if curve.IsClosed():
            while d1 < 0: d1 += perim
            while d1 > perim: d1 -= perim
        p = curve.PerimToPoint(d1)
        curve.Break(p)

    def get_z_at_perim(self, current_perim, curve, radius, start_depth, depth, final_depth):
        # return the z for this position on the kurve ( specified by current_perim ), for this tag
        # if the position is not within the tag, then depth is returned
        cut_depth = start_depth - depth
        half_flat_top = radius + self.width / 2

        z = depth
        d = curve.PointToPerim(self.p)
        dist_from_d = math.fabs(current_perim - d)
        if dist_from_d < half_flat_top:
            # on flat top of tag
            z = final_depth + self.height
        elif dist_from_d < half_flat_top + self.ramp_width:
            # on ramp
            dist_up_ramp = (half_flat_top + self.ramp_width) - dist_from_d
            z = final_depth + dist_up_ramp * math.tan(self.angle)
        if z < depth: z = depth
        return z

    def dist(self, curve):
        # return the distance from the tag point to the given kurve
        d = curve.PointToPerim(self.p)
        p = curve.PerimToPoint(d)
        v = self.p - p
        return v.length()

tags = []

def clear_tags():
    global tags
    tags = []

def add_tag(p, width, angle, height):
    global tags
    tag = Tag(p, width, angle, height)
    tags.append(tag)

def split_for_tags( curve, radius, start_depth, depth, final_depth ):
    global tags
    for tag in tags:
        tag.split_curve(curve, radius, start_depth, depth, final_depth)

def get_tag_z_for_span(current_perim, curve, radius, start_depth, depth, final_depth):
    global tags
    max_z = None
    perim = curve.Perim()
    for tag in tags:
        z = tag.get_z_at_perim(current_perim, curve, radius, start_depth, depth, final_depth)
        if max_z == None or z > max_z:
            max_z = z
        if curve.IsClosed():
            # do the same test, wrapped around the closed kurve
            z = tag.get_z_at_perim(current_perim - perim, curve, radius, start_depth, depth, final_depth)
            if max_z == None or z > max_z:
                max_z = z
            z = tag.get_z_at_perim(current_perim + perim, curve, radius, start_depth, depth, final_depth)
            if max_z == None or z > max_z:
                max_z = z

    return max_z

def add_roll_on(curve, roll_on_curve, direction, roll_radius, offset_extra, roll_on):
    if direction == "on": roll_on = None
    if curve.getNumVertices() <= 1: return
    first_span = curve.GetFirstSpan()

    if roll_on == None:
        rollstart = first_span.p
    elif roll_on == 'auto':
        v = first_span.GetVector(0.0)
        if direction == 'right':
            off_v = area.Point(v.y, -v.x)
        else:
            off_v = area.Point(-v.y, v.x)

        rollstart = first_span.p + off_v * roll_radius - v * roll_radius
        global crc_dir
        if '-' in str(off_v.x):
            crc_dir = "inside"
        else:
            crc_dir = "outside"
    else:
        rollstart = roll_on

    rvertex = area.Vertex(first_span.p)

    if first_span.p == rollstart:
        rvertex.type = 0
    else:
        v = first_span.GetVector(0.0) # get start direction
        rvertex.c, rvertex.type = area.TangentialArc(first_span.p, rollstart, -v)
        rvertex.type = -rvertex.type # because TangentialArc was used in reverse

    # add a start roll on point
    roll_on_curve.append(rollstart)

    # add the roll on arc
    roll_on_curve.append(rvertex)

def add_roll_off(curve, roll_off_curve, direction, roll_radius, offset_extra, roll_off):
    if direction == "on": return
    if roll_off == None: return
    if curve.getNumVertices() <= 1: return

    last_span = curve.GetLastSpan()

    if roll_off == 'auto':
        v = last_span.GetVector(1.0) # get end direction
        if direction == 'right':
            off_v = area.Point(v.y, -v.x)
        else:
            off_v = area.Point(-v.y, v.x)

        rollend = last_span.v.p + off_v * roll_radius + v * roll_radius;
    else:
        rollend =  roll_off

    # add the end of the original kurve
    roll_off_curve.append(last_span.v.p)
    if rollend == last_span.v.p: return
    rvertex = area.Vertex(rollend)
    v = last_span.GetVector(1.0) # get end direction
    rvertex.c, rvertex.type = area.TangentialArc(last_span.v.p, rollend, v)

    # add the roll off arc
    roll_off_curve.append(rvertex)

def cut_curve(curve):
    for span in curve.GetSpans():
        if span.v.type == 0:#line
            feed(span.v.p.x, span.v.p.y)
        else:
            c = span.v.c - span.p # make relative to the start position
            if span.v.type == 1:# anti-clockwise arc
                arc_ccw(span.v.p.x, span.v.p.y, i = c.x, j = c.y)
            else:
                arc_cw(span.v.p.x, span.v.p.y, i = c.x, j = c.y)

def add_CRC_start_line(curve, radius):
    # to do
    pass

# profile command,
# direction should be 'left' or 'right' or 'on'
def profile(curve, direction = "on", radius = 1.0, offset_extra = 0.0, roll_radius = 2.0, roll_on = None, roll_off = None, rapid_down_to_height = None, clearance = None, start_depth = None, step_down = None, final_depth = None):
    global tags

    offset_curve = area.Curve(curve)

    if direction != "on":
        if direction != "left" and direction != "right":
            raise "direction must be left or right", direction

        # get tool diameter
        offset = radius + offset_extra
        if use_CRC() == False:
            if direction == "right":
                offset = -offset
            offset_success = offset_curve.Offset(offset)
            if offset_success == False:
                raise "couldn't offset kurve " + offset_curve

    # remove tags further than radius from the offset kurve
    new_tags = []
    for tag in tags:
        if tag.dist(offset_curve) <= radius + 0.001:
            new_tags.append(tag)
    tags = new_tags

    if offset_curve.getNumVertices() <= 1:
        raise "sketch has no spans!"

    # do multiple depths
    total_to_cut = start_depth - final_depth;
    num_step_downs = int(float(total_to_cut) / math.fabs(step_down) + 0.999999)

    # tags
    if len(tags) > 0:
        # make a copy to restore to after each level
        copy_of_offset_curve = area.Curve(offset_curve)

    prev_depth = start_depth
    for step in range(0, num_step_downs):
        depth_of_cut = ( start_depth - final_depth ) * ( step + 1 ) / num_step_downs
        depth = start_depth - depth_of_cut
        mat_depth = prev_depth

        if len(tags) > 0:
            split_for_tags(offset_curve, radius, start_depth, depth, final_depth)

        # make the roll on and roll off kurves
        roll_on_curve = area.Curve()
        add_roll_on(offset_curve, roll_on_curve, direction, roll_radius, offset_extra, roll_on)
        roll_off_curve = area.Curve()
        add_roll_off(offset_curve, roll_off_curve, direction, roll_radius, offset_extra, roll_off)

        # get the tag depth at the start
        start_z = get_tag_z_for_span(0, offset_curve, radius, start_depth, depth, final_depth)
        if start_z > mat_depth: mat_depth = start_z

        # rapid across to the start
        s = roll_on_curve.FirstVertex().p
        # crc_start point
        crc_diameter = radius*2
        if use_CRC():
            if crc_dir == "inside":
                if direction == "right":
                    rapid(s.x+crc_diameter, s.y-crc_diameter)
                else:
                    rapid(s.x-crc_diameter, s.y-crc_diameter)
            else:
                if direction == "right":
                    rapid(s.x-crc_diameter, s.y+crc_diameter)
                else:
                    rapid(s.x+crc_diameter, s.y+crc_diameter)

        else:
            rapid(s.x, s.y)

        # rapid down to just above the material
        rapid(z = mat_depth + rapid_down_to_height)

        # feed down to depth
        mat_depth = depth
        if start_z > mat_depth: mat_depth = start_z
        feed(z = mat_depth)

        if use_CRC():
            start_CRC(direction == "left", radius)
            # move to the startpoint
            feed(s.x, s.y)

        # cut the roll on arc
        cut_curve(roll_on_curve)

        # cut the main kurve
        current_perim = 0.0

        for span in offset_curve.GetSpans():
            # height for tags
            current_perim += span.Length()
            ez = get_tag_z_for_span(current_perim, offset_curve, radius, start_depth, depth, final_depth)

            if span.v.type == 0:#line
                feed(span.v.p.x, span.v.p.y, ez)
            else:
                c = span.v.c - span.p # make relative to the start position
                if span.v.type == 1:# anti-clockwise arc
                    arc_ccw(span.v.p.x, span.v.p.y, ez, i = c.x, j = c.y)
                else:
                    arc_cw(span.v.p.x, span.v.p.y, ez, i = c.x, j = c.y)

        # cut the roll off arc
        cut_curve(roll_off_curve)

        # restore the unsplit kurve
        if len(tags) > 0:
            offset_curve = area.Curve(copy_of_offset_curve)

        # rapid up to the clearance height
        rapid(z = clearance)

        if use_CRC():
            end_CRC()

    del offset_curve

    if len(tags) > 0:
        del copy_of_offset_curve
<commit_msg>applied IchGuckLive's CRC changes<commit_after>import math
from nc.nc import *
import area

def make_smaller( curve, start = None, finish = None ):
    if start != None:
        curve.ChangeStart(start)

    if finish != None:
        curve.ChangeEnd(finish)

class Tag:
    def __init__(self, p, width, angle, height):
        self.p = p
        self.width = width # measured at the top of the tag. In the toolpath, the tag width will be this with plus the tool diameter, so that the finished tag has this "width" at it's smallest
        self.angle = angle # the angle of the ramp in radians. Between 0 and Pi/2; 0 is horizontal, Pi/2 is vertical
        self.height = height # the height of the tag, always measured above "final_depth"
        self.ramp_width = self.height / math.tan(self.angle)

    def split_curve(self, curve, radius, start_depth, depth, final_depth):
        tag_top_depth = final_depth + self.height

        if depth > tag_top_depth - 0.0000001:
            return # kurve is above this tag, so doesn't need splitting

        height_above_depth = tag_top_depth - depth
        ramp_width_at_depth = height_above_depth / math.tan(self.angle)
        cut_depth = start_depth - depth
        half_flat_top = radius + self.width / 2

        d = curve.PointToPerim(self.p)
        d0 = d - half_flat_top
        perim = curve.Perim()
        if curve.IsClosed():
            while d0 < 0: d0 += perim
            while d0 > perim: d0 -= perim
        p = curve.PerimToPoint(d0)
        curve.Break(p)
        d1 = d + half_flat_top
        if curve.IsClosed():
            while d1 < 0: d1 += perim
            while d1 > perim: d1 -= perim
        p = curve.PerimToPoint(d1)
        curve.Break(p)

        d0 = d - half_flat_top - ramp_width_at_depth
        if curve.IsClosed():
            while d0 < 0: d0 += perim
            while d0 > perim: d0 -= perim
        p = curve.PerimToPoint(d0)
        curve.Break(p)
        d1 = d + half_flat_top + ramp_width_at_depth
        if curve.IsClosed():
            while d1 < 0: d1 += perim
            while d1 > perim: d1 -= perim
        p = curve.PerimToPoint(d1)
        curve.Break(p)

    def get_z_at_perim(self, current_perim, curve, radius, start_depth, depth, final_depth):
        # return the z for this position on the kurve ( specified by current_perim ), for this tag
        # if the position is not within the tag, then depth is returned
        cut_depth = start_depth - depth
        half_flat_top = radius + self.width / 2

        z = depth
        d = curve.PointToPerim(self.p)
        dist_from_d = math.fabs(current_perim - d)
        if dist_from_d < half_flat_top:
            # on flat top of tag
            z = final_depth + self.height
        elif dist_from_d < half_flat_top + self.ramp_width:
            # on ramp
            dist_up_ramp = (half_flat_top + self.ramp_width) - dist_from_d
            z = final_depth + dist_up_ramp * math.tan(self.angle)
        if z < depth: z = depth
        return z

    def dist(self, curve):
        # return the distance from the tag point to the given kurve
        d = curve.PointToPerim(self.p)
        p = curve.PerimToPoint(d)
        v = self.p - p
        return v.length()

tags = []

def clear_tags():
    global tags
    tags = []

def add_tag(p, width, angle, height):
    global tags
    tag = Tag(p, width, angle, height)
    tags.append(tag)

def split_for_tags( curve, radius, start_depth, depth, final_depth ):
    global tags
    for tag in tags:
        tag.split_curve(curve, radius, start_depth, depth, final_depth)

def get_tag_z_for_span(current_perim, curve, radius, start_depth, depth, final_depth):
    global tags
    max_z = None
    perim = curve.Perim()
    for tag in tags:
        z = tag.get_z_at_perim(current_perim, curve, radius, start_depth, depth, final_depth)
        if max_z == None or z > max_z:
            max_z = z
        if curve.IsClosed():
            # do the same test, wrapped around the closed kurve
            z = tag.get_z_at_perim(current_perim - perim, curve, radius, start_depth, depth, final_depth)
            if max_z == None or z > max_z:
                max_z = z
            z = tag.get_z_at_perim(current_perim + perim, curve, radius, start_depth, depth, final_depth)
            if max_z == None or z > max_z:
                max_z = z

    return max_z

def add_roll_on(curve, roll_on_curve, direction, roll_radius, offset_extra, roll_on):
    if direction == "on": roll_on = None
    if curve.getNumVertices() <= 1: return
    first_span = curve.GetFirstSpan()

    if roll_on == None:
        rollstart = first_span.p
    elif roll_on == 'auto':
        v = first_span.GetVector(0.0)
        if direction == 'right':
            off_v = area.Point(v.y, -v.x)
        else:
            off_v = area.Point(-v.y, v.x)
        rollstart = first_span.p + off_v * roll_radius - v * roll_radius
    else:
        rollstart = roll_on

    rvertex = area.Vertex(first_span.p)

    if first_span.p == rollstart:
        rvertex.type = 0
    else:
        v = first_span.GetVector(0.0) # get start direction
        rvertex.c, rvertex.type = area.TangentialArc(first_span.p, rollstart, -v)
        rvertex.type = -rvertex.type # because TangentialArc was used in reverse
    # add a start roll on point
    roll_on_curve.append(rollstart)

    # add the roll on arc
    roll_on_curve.append(rvertex)

def add_roll_off(curve, roll_off_curve, direction, roll_radius, offset_extra, roll_off):
    if direction == "on": return
    if roll_off == None: return
    if curve.getNumVertices() <= 1: return

    last_span = curve.GetLastSpan()

    if roll_off == 'auto':
        v = last_span.GetVector(1.0) # get end direction
        if direction == 'right':
            off_v = area.Point(v.y, -v.x)
        else:
            off_v = area.Point(-v.y, v.x)

        rollend = last_span.v.p + off_v * roll_radius + v * roll_radius;
    else:
        rollend =  roll_off

    # add the end of the original kurve
    roll_off_curve.append(last_span.v.p)
    if rollend == last_span.v.p: return
    rvertex = area.Vertex(rollend)
    v = last_span.GetVector(1.0) # get end direction
    rvertex.c, rvertex.type = area.TangentialArc(last_span.v.p, rollend, v)

    # add the roll off arc
    roll_off_curve.append(rvertex)

def cut_curve(curve):
    for span in curve.GetSpans():
        if span.v.type == 0:#line
            feed(span.v.p.x, span.v.p.y)
        else:
            c = span.v.c - span.p # make relative to the start position
            if span.v.type == 1:# anti-clockwise arc
                arc_ccw(span.v.p.x, span.v.p.y, i = c.x, j = c.y)
            else:
                arc_cw(span.v.p.x, span.v.p.y, i = c.x, j = c.y)

def add_CRC_start_line(curve,roll_on_curve,roll_off_curve,radius,direction,crc_start_point):
    first_span = curve.GetFirstSpan()
    v = first_span.GetVector(0.0)
    if direction == 'right':
        off_v = area.Point(v.y, -v.x)
    else:
        off_v = area.Point(-v.y, v.x)
    startpoint_roll_on = roll_on_curve.FirstVertex().p
    crc_start = startpoint_roll_on + off_v * radius + v * radius
    crc_start_point.x = crc_start.x
    crc_start_point.y = crc_start.y

# profile command,
# direction should be 'left' or 'right' or 'on'
def profile(curve, direction = "on", radius = 1.0, offset_extra = 0.0, roll_radius = 2.0, roll_on = None, roll_off = None, rapid_down_to_height = None, clearance = None, start_depth = None, step_down = None, final_depth = None):
    global tags

    offset_curve = area.Curve(curve)

    if direction != "on":
        if direction != "left" and direction != "right":
            raise "direction must be left or right", direction

        # get tool diameter
        offset = radius + offset_extra
        if use_CRC() == False:
            if direction == "right":
                offset = -offset
            offset_success = offset_curve.Offset(offset)
            if offset_success == False:
                raise "couldn't offset kurve " + offset_curve

    # remove tags further than radius from the offset kurve
    new_tags = []
    for tag in tags:
        if tag.dist(offset_curve) <= radius + 0.001:
            new_tags.append(tag)
    tags = new_tags

    if offset_curve.getNumVertices() <= 1:
        raise "sketch has no spans!"

    # do multiple depths
    total_to_cut = start_depth - final_depth;
    num_step_downs = int(float(total_to_cut) / math.fabs(step_down) + 0.999999)

    # tags
    if len(tags) > 0:
        # make a copy to restore to after each level
        copy_of_offset_curve = area.Curve(offset_curve)

    prev_depth = start_depth
    for step in range(0, num_step_downs):
        depth_of_cut = ( start_depth - final_depth ) * ( step + 1 ) / num_step_downs
        depth = start_depth - depth_of_cut
        mat_depth = prev_depth

        if len(tags) > 0:
            split_for_tags(offset_curve, radius, start_depth, depth, final_depth)

        # make the roll on and roll off kurves
        roll_on_curve = area.Curve()
        add_roll_on(offset_curve, roll_on_curve, direction, roll_radius, offset_extra, roll_on)
        roll_off_curve = area.Curve()
        add_roll_off(offset_curve, roll_off_curve, direction, roll_radius, offset_extra, roll_off)
        if use_CRC():
            crc_start_point = area.Point()
            add_CRC_start_line(offset_curve,roll_on_curve,roll_off_curve,radius,direction,crc_start_point)

        # get the tag depth at the start
        start_z = get_tag_z_for_span(0, offset_curve, radius, start_depth, depth, final_depth)
        if start_z > mat_depth: mat_depth = start_z

        # rapid across to the start
        s = roll_on_curve.FirstVertex().p

        # start point
        if use_CRC():
            rapid(crc_start_point.x,crc_start_point.y)
        else:
            rapid(s.x, s.y)

        # rapid down to just above the material
        rapid(z = mat_depth + rapid_down_to_height)

        # feed down to depth
        mat_depth = depth
        if start_z > mat_depth: mat_depth = start_z
        feed(z = mat_depth)

        if use_CRC():
            start_CRC(direction == "left", radius)
            # move to the startpoint
            feed(s.x, s.y)

        # cut the roll on arc
        cut_curve(roll_on_curve)

        # cut the main kurve
        current_perim = 0.0

        for span in offset_curve.GetSpans():
            # height for tags
            current_perim += span.Length()
            ez = get_tag_z_for_span(current_perim, offset_curve, radius, start_depth, depth, final_depth)

            if span.v.type == 0:#line
                feed(span.v.p.x, span.v.p.y, ez)
            else:
                c = span.v.c - span.p # make relative to the start position
                if span.v.type == 1:# anti-clockwise arc
                    arc_ccw(span.v.p.x, span.v.p.y, ez, i = c.x, j = c.y)
                else:
                    arc_cw(span.v.p.x, span.v.p.y, ez, i = c.x, j = c.y)

        # cut the roll off arc
        cut_curve(roll_off_curve)

        # restore the unsplit kurve
        if len(tags) > 0:
            offset_curve = area.Curve(copy_of_offset_curve)

        # rapid up to the clearance height
        rapid(z = clearance)

        if use_CRC():
            end_CRC()

    del offset_curve

    if len(tags) > 0:
        del copy_of_offset_curve
<|endoftext|>