<commit_before>#!/usr/bin/env python

"""Report state based on FAUCET/Gauge/Prometheus variables."""

# TODO: this script and is usage is experimental and its output
# is expected to change significantly.
# TODO: add control functionality.

# Copyright (C) 2015 Brad Cowie, Christopher Lorier and Joe Stringer.
# Copyright (C) 2015 Research and Education Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2017 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# pytype: disable=pyi-error
# pytype: disable=import-error
import getopt
import sys
import time
import urllib.request, urllib.parse, urllib.error
import requests
from prometheus_client import parser

# TODO: byte/packet counters could be per second (given multiple samples)
VAL_DECODE = {
    'learned_macs': lambda mac: "{:012x}".format(int(mac))
}

def scrape_prometheus(endpoints, retries=3):
    """Scrape a list of Prometheus/FAUCET/Gauge endpoints and aggregate results."""
    metrics = []
    for endpoint in endpoints:
        content = None
        err = None
        for _ in range(retries):
            try:
                if endpoint.startswith('http'):
                    response = requests.get(endpoint)
                    if response.status_code == requests.status_codes.codes.ok: # pylint: disable=no-member
                        content = response.content.decode('utf-8', 'strict')
                        break
                else:
                    response = urllib.request.urlopen(endpoint) # pytype: disable=module-attr
                    content = response.read().decode('utf-8', 'strict')
                    break
            except requests.exceptions.ConnectionError as exception:
                err = exception
                time.sleep(1)
        if err is not None:
            print(err)
            sys.exit(1)
        endpoint_metrics = parser.text_string_to_metric_families(
            content)
        metrics.extend(endpoint_metrics)
    return metrics

def report_label_match_metrics(report_metrics, metrics,
                               nonzero_only=False, delim='\t', label_matches=None):
    """Text report on a list of Prometheus metrics."""
    for metric in metrics:
        if not report_metrics or metric.name in report_metrics:
            for _, labels, value in metric.samples:
                if label_matches is None or \
                    (label_matches and set(label_matches.items()).issubset(set(labels.items()))):
                    if nonzero_only and int(value) == 0:
                        continue
                    try:
                        value = VAL_DECODE[metric.name](value)
                    except KeyError:
                        pass
                    sorted_labels = [(k, v) for k, v in sorted(labels.items())]
                    print((delim.join((metric.name, str(sorted_labels), str(value)))))

def usage():
    usage_vars = {'self': sys.argv[0]}
    print(("""
Retrieve FAUCET/Gauge state using Prometheus variables.

    {self} [-n] <-e|--endpoints=http://server:port> [-m|--metrics=prometheus_metrics,] [-l|--labels=name:value,]

    -n: Don't report 0 values
    -e|--endpoints: list of Prometheus endpoints to query (comma separated)
    -m|--metrics: list of Prometheus variables to query (comma separated)
    -l|--labels: filter list of Prometheus variables by labels that must be present (comma separated)

Examples:

    MACs learned on a DP.

    {self} -n --endpoints=http://172.17.0.1:9302 --metrics=learned_macs --labels=dp_id:0xb827eb608918

    Status of all DPs

    {self} -n --endpoints=http://172.17.0.1:9302 --metrics=dp_status
""".format(**usage_vars))) # pytype: disable=duplicate-keyword-argument
    sys.exit(-1)

def main():
    try:
        opts, _ = getopt.getopt(
            sys.argv[1:], 'ne:m:l:', ['nonzero', 'endpoints=', 'metrics=', 'labels='])
    except getopt.GetoptError:
        usage()

    endpoints = []
    report_metrics = []
    label_matches = None
    nonzero_only = False

    for opt, arg in opts:
        if opt in ('-n', '--nonzero'):
            nonzero_only = True
        elif opt in ('-e', '--endpoints'):
            endpoints = arg.split(',')
        elif opt in ('-m', '--metrics'):
            report_metrics = arg.split(',')
        elif opt in ('-l', '--labels'):
            for label_value in arg.split(','):
                label, value = label_value.split(':')
                if label_matches is None:
                    label_matches = {}
                label_matches[label] = value
        else:
            usage()

    metrics = scrape_prometheus(endpoints)
    report_label_match_metrics(
        report_metrics, metrics, nonzero_only=nonzero_only, label_matches=label_matches)

if __name__ == '__main__':
    main()
<commit_msg>Add colons between octets in mac addresses.<commit_after>#!/usr/bin/env python

"""Report state based on FAUCET/Gauge/Prometheus variables."""

# TODO: this script and is usage is experimental and its output
# is expected to change significantly.
# TODO: add control functionality.

# Copyright (C) 2015 Brad Cowie, Christopher Lorier and Joe Stringer.
# Copyright (C) 2015 Research and Education Advanced Network New Zealand Ltd.
# Copyright (C) 2015--2017 The Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# pytype: disable=pyi-error
# pytype: disable=import-error
import getopt
import sys
import time
import urllib.request, urllib.parse, urllib.error
import requests
from prometheus_client import parser

# TODO: byte/packet counters could be per second (given multiple samples)
VAL_DECODE = {
    'learned_macs': lambda mac: ':'.join(format(octet, '02x') for octet in int(mac).to_bytes(6, byteorder='big'))
}

def scrape_prometheus(endpoints, retries=3):
    """Scrape a list of Prometheus/FAUCET/Gauge endpoints and aggregate results."""
    metrics = []
    for endpoint in endpoints:
        content = None
        err = None
        for _ in range(retries):
            try:
                if endpoint.startswith('http'):
                    response = requests.get(endpoint)
                    if response.status_code == requests.status_codes.codes.ok: # pylint: disable=no-member
                        content = response.content.decode('utf-8', 'strict')
                        break
                else:
                    response = urllib.request.urlopen(endpoint) # pytype: disable=module-attr
                    content = response.read().decode('utf-8', 'strict')
                    break
            except requests.exceptions.ConnectionError as exception:
                err = exception
                time.sleep(1)
        if err is not None:
            print(err)
            sys.exit(1)
        endpoint_metrics = parser.text_string_to_metric_families(
            content)
        metrics.extend(endpoint_metrics)
    return metrics

def report_label_match_metrics(report_metrics, metrics,
                               nonzero_only=False, delim='\t', label_matches=None):
    """Text report on a list of Prometheus metrics."""
    for metric in metrics:
        if not report_metrics or metric.name in report_metrics:
            for _, labels, value in metric.samples:
                if label_matches is None or \
                    (label_matches and set(label_matches.items()).issubset(set(labels.items()))):
                    if nonzero_only and int(value) == 0:
                        continue
                    try:
                        value = VAL_DECODE[metric.name](value)
                    except KeyError:
                        pass
                    sorted_labels = [(k, v) for k, v in sorted(labels.items())]
                    print((delim.join((metric.name, str(sorted_labels), str(value)))))

def usage():
    usage_vars = {'self': sys.argv[0]}
    print(("""
Retrieve FAUCET/Gauge state using Prometheus variables.

    {self} [-n] <-e|--endpoints=http://server:port> [-m|--metrics=prometheus_metrics,] [-l|--labels=name:value,]

    -n: Don't report 0 values
    -e|--endpoints: list of Prometheus endpoints to query (comma separated)
    -m|--metrics: list of Prometheus variables to query (comma separated)
    -l|--labels: filter list of Prometheus variables by labels that must be present (comma separated)

Examples:

    MACs learned on a DP.

    {self} -n --endpoints=http://172.17.0.1:9302 --metrics=learned_macs --labels=dp_id:0xb827eb608918

    Status of all DPs

    {self} -n --endpoints=http://172.17.0.1:9302 --metrics=dp_status
""".format(**usage_vars))) # pytype: disable=duplicate-keyword-argument
    sys.exit(-1)

def main():
    try:
        opts, _ = getopt.getopt(
            sys.argv[1:], 'ne:m:l:', ['nonzero', 'endpoints=', 'metrics=', 'labels='])
    except getopt.GetoptError:
        usage()

    endpoints = []
    report_metrics = []
    label_matches = None
    nonzero_only = False

    for opt, arg in opts:
        if opt in ('-n', '--nonzero'):
            nonzero_only = True
        elif opt in ('-e', '--endpoints'):
            endpoints = arg.split(',')
        elif opt in ('-m', '--metrics'):
            report_metrics = arg.split(',')
        elif opt in ('-l', '--labels'):
            for label_value in arg.split(','):
                label, value = label_value.split(':')
                if label_matches is None:
                    label_matches = {}
                label_matches[label] = value
        else:
            usage()

    metrics = scrape_prometheus(endpoints)
    report_label_match_metrics(
        report_metrics, metrics, nonzero_only=nonzero_only, label_matches=label_matches)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
from itertools import combinations
import csv
import exampleIO
import dedupe
import os
import time
import optparse
import logging

optp = optparse.OptionParser()
optp.add_option('-v', '--verbose', dest='verbose', action='count',
                help='Increase verbosity (specify multiple times for more)'
                )
(opts, args) = optp.parse_args()
log_level = logging.WARNING
if opts.verbose == 1:
    log_level = logging.INFO
elif opts.verbose >= 2:
    log_level = logging.DEBUG
logging.getLogger().setLevel(log_level)

#logging.basicConfig(level=log_level)

def canonicalImport(filename):
    preProcess = exampleIO.preProcess

    data_d = {}

    with open(filename) as f:
        reader = csv.DictReader(f)
        for (i, row) in enumerate(reader):
            clean_row = [(k, preProcess(v)) for (k, v) in
                         row.iteritems()]
            data_d[i] = dedupe.core.frozendict(clean_row)

    return data_d, reader.fieldnames

def evaluateDuplicates(found_dupes, true_dupes):
    true_positives = found_dupes.intersection(true_dupes)
    false_positives = found_dupes.difference(true_dupes)
    uncovered_dupes = true_dupes.difference(found_dupes)

    print 'found duplicate'
    print len(found_dupes)

    print 'precision'
    print 1 - len(false_positives) / float(len(found_dupes))

    print 'recall'
    print len(true_positives) / float(len(true_dupes))

settings_file = 'canonical_learned_settings.json'
raw_data = 'tests/datasets/restaurant-nophone-training.csv'

data_d, header = canonicalImport(raw_data)

training_pairs = dedupe.trainingDataDedupe(data_d,
                                           'unique_id',
                                           5000)

duplicates_s = set(frozenset(pair) for pair in training_pairs['match'])

t0 = time.time()

print 'number of known duplicate pairs', len(duplicates_s)

if os.path.exists(settings_file):
    deduper = dedupe.StaticDedupe(settings_file)
else:
    fields = {'name': {'type': 'String'},
              'address': {'type': 'String'},
              'cuisine': {'type': 'ShortString'},
              'city' : {'type' : 'ShortString'}
              }

    deduper = dedupe.Dedupe(fields)
    deduper.sample(data_d, 1000000)
    deduper.markPairs(training_pairs)
    deduper.train()
    deduper.writeSettings(settings_file)

alpha = deduper.threshold(data_d)

# print candidates
print 'clustering...'
clustered_dupes = deduper.match(data_d, threshold=alpha)

print 'Evaluate Scoring'
found_dupes = set([frozenset((data_d[pair[0]], data_d[pair[1]]))
                   for (pair, score) in deduper.matches
                   if score > alpha])

evaluateDuplicates(found_dupes, duplicates_s)

print 'Evaluate Clustering'

confirm_dupes = set([])
for dupe_set in clustered_dupes:
    for pair in combinations(dupe_set, 2):
        confirm_dupes.add(frozenset((data_d[pair[0]], data_d[pair[1]])))

evaluateDuplicates(confirm_dupes, duplicates_s)

print 'ran in ', time.time() - t0, 'seconds'
<commit_msg>Updated canonical test to read and write settings correctly<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
from itertools import combinations
import csv
import exampleIO
import dedupe
import os
import time
import optparse
import logging

optp = optparse.OptionParser()
optp.add_option('-v', '--verbose', dest='verbose', action='count',
                help='Increase verbosity (specify multiple times for more)'
                )
(opts, args) = optp.parse_args()
log_level = logging.WARNING
if opts.verbose == 1:
    log_level = logging.INFO
elif opts.verbose >= 2:
    log_level = logging.DEBUG
logging.getLogger().setLevel(log_level)

#logging.basicConfig(level=log_level)

def canonicalImport(filename):
    preProcess = exampleIO.preProcess

    data_d = {}

    with open(filename) as f:
        reader = csv.DictReader(f)
        for (i, row) in enumerate(reader):
            clean_row = [(k, preProcess(v)) for (k, v) in
                         row.iteritems()]
            data_d[i] = dedupe.core.frozendict(clean_row)

    return data_d, reader.fieldnames

def evaluateDuplicates(found_dupes, true_dupes):
    true_positives = found_dupes.intersection(true_dupes)
    false_positives = found_dupes.difference(true_dupes)
    uncovered_dupes = true_dupes.difference(found_dupes)

    print 'found duplicate'
    print len(found_dupes)

    print 'precision'
    print 1 - len(false_positives) / float(len(found_dupes))

    print 'recall'
    print len(true_positives) / float(len(true_dupes))

settings_file = 'canonical_learned_settings.json'
raw_data = 'tests/datasets/restaurant-nophone-training.csv'

data_d, header = canonicalImport(raw_data)

training_pairs = dedupe.trainingDataDedupe(data_d,
                                           'unique_id',
                                           5000)

duplicates_s = set(frozenset(pair) for pair in training_pairs['match'])

t0 = time.time()

print 'number of known duplicate pairs', len(duplicates_s)

if os.path.exists(settings_file):
    with open(settings_file, 'rb') as f:
        deduper = dedupe.StaticDedupe(f)
else:
    fields = {'name': {'type': 'String'},
              'address': {'type': 'String'},
              'cuisine': {'type': 'ShortString'},
              'city' : {'type' : 'ShortString'}
              }

    deduper = dedupe.Dedupe(fields)
    deduper.sample(data_d, 1000000)
    deduper.markPairs(training_pairs)
    deduper.train()
    with open(settings_file, 'wb') as f:
        deduper.writeSettings(f)

alpha = deduper.threshold(data_d)

# print candidates
print 'clustering...'
clustered_dupes = deduper.match(data_d, threshold=alpha)

print 'Evaluate Scoring'
found_dupes = set([frozenset((data_d[pair[0]], data_d[pair[1]]))
                   for (pair, score) in deduper.matches
                   if score > alpha])

evaluateDuplicates(found_dupes, duplicates_s)

print 'Evaluate Clustering'

confirm_dupes = set([])
for dupe_set in clustered_dupes:
    for pair in combinations(dupe_set, 2):
        confirm_dupes.add(frozenset((data_d[pair[0]], data_d[pair[1]])))

evaluateDuplicates(confirm_dupes, duplicates_s)

print 'ran in ', time.time() - t0, 'seconds'
<|endoftext|>


<commit_before>"""
============
rdflib.store
============

``Context-aware``: An RDF store capable of storing statements within contexts
is considered context-aware. Essentially, such a store is able to partition
the RDF model it represents into individual, named, and addressable
sub-graphs.

Relevant Notation3 reference regarding formulae, quoted statements, and such:
http://www.w3.org/DesignIssues/Notation3.html

``Formula-aware``: An RDF store capable of distinguishing between statements
that are asserted and statements that are quoted is considered formula-aware.

``Conjunctive Graph``: This refers to the 'top-level' Graph. It is the
aggregation of all the contexts within it and is also the appropriate,
absolute boundary for closed world assumptions / models.

For the sake of persistence, Conjunctive Graphs must be distinguished by
identifiers (that may not necessarily be RDF identifiers or may be an RDF
identifier normalized - SHA1/MD5 perhaps - for database naming purposes ).

``Conjunctive Query``: Any query that doesn't limit the store to search
within a named context only. Such a query expects a context-aware store to
search the entire asserted universe (the conjunctive graph). A formula-aware
store is expected not to include quoted statements when matching such a query.
"""

#Constants representing the state of a Store (returned by the open method)
VALID_STORE     = 1
CORRUPTED_STORE = 0
NO_STORE        = -1
UNKNOWN         = None

from rdflib.events import Dispatcher, Event

class StoreCreatedEvent(Event):
    """
    This event is fired when the Store is created, it has the folloing attribute:

      - 'configuration' string that is used to create the store

    """

class TripleAddedEvent(Event):
    """
    This event is fired when a triple is added, it has the following attributes:

      - 'triple' added to the graph
      - 'context' of the triple if any
      - 'graph' that the triple was added to
    """

class TripleRemovedEvent(Event):
    """
    This event is fired when a triple is removed, it has the following attributes:

      - 'triple' removed from the graph
      - 'context' of the triple if any
      - 'graph' that the triple was removed from
    """

from cPickle import Pickler, Unpickler, UnpicklingError
from cStringIO import StringIO

class NodePickler(object):
    def __init__(self):
        self._objects = {}
        self._ids = {}
        self._get_object = self._objects.__getitem__

    def _get_ids(self, key):
        try:
            return self._ids.get(key)
        except TypeError, e:
            return None

    def register(self, object, id):
        self._objects[id] = object
        self._ids[object] = id

    def loads(self, s):
        up = Unpickler(StringIO(s))
        up.persistent_load = self._get_object
        try:
            return up.load()
        except KeyError, e:
            raise UnpicklingError, "Could not find Node class for %s" % e

    def dumps(self, obj, protocol=None, bin=None):
        src = StringIO()
        p = Pickler(src)
        p.persistent_id = self._get_ids
        p.dump(obj)
        return src.getvalue()

class Store(object):
    #Properties
    context_aware = False
    formula_aware = False
    transaction_aware = False
    batch_unification = False
    def __init__(self, configuration=None, identifier=None):
        """
        identifier: URIRef of the Store. Defaults to CWD
        configuration: string containing infomation open can use to
        connect to datastore.
        """
        self.__node_pickler = None
        self.dispatcher = Dispatcher()
        if configuration:
            self.open(configuration)

    def __get_node_pickler(self):
        if self.__node_pickler is None:
            from rdflib.term import URIRef
            from rdflib.term import BNode
            from rdflib.term import Literal
            from rdflib.graph import Graph, QuotedGraph, GraphValue
            from rdflib.term import Variable
            from rdflib.term import Statement
            self.__node_pickler = np = NodePickler()
            np.register(self, "S")
            np.register(URIRef, "U")
            np.register(BNode, "B")
            np.register(Literal, "L")
            np.register(Graph, "G")
            np.register(QuotedGraph, "Q")
            np.register(Variable, "V")
            np.register(Statement, "s")
            np.register(GraphValue, "v")
        return self.__node_pickler
    node_pickler = property(__get_node_pickler)

    #Database management methods
    def create(self, configuration):
        self.dispatcher.dispatch(StoreCreatedEvent(configuration=configuration))

    def open(self, configuration, create=False):
        """
        Opens the store specified by the configuration string. If
        create is True a store will be created if it does not already
        exist. If create is False and a store does not already exist
        an exception is raised. An exception is also raised if a store
        exists, but there is insufficient permissions to open the
        store.  This should return one of VALID_STORE,CORRUPTED_STORE,or NO_STORE
        """
        return UNKNOWN

    def close(self, commit_pending_transaction=False):
        """
        This closes the database connection. The commit_pending_transaction parameter specifies whether to
        commit all pending transactions before closing (if the store is transactional).
        """

    def destroy(self, configuration):
        """
        This destroys the instance of the store identified by the configuration string.
        """

    def gc(self):
        """
        Allows the store to perform any needed garbage collection
        """
        pass

    #RDF APIs
    def add(self, (subject, predicate, object), context, quoted=False):
        """
        Adds the given statement to a specific context or to the model. The quoted argument
        is interpreted by formula-aware stores to indicate this statement is quoted/hypothetical
        It should be an error to not specify a context and have the quoted argument be True.
        It should also be an error for the quoted argument to be True when the store is not formula-aware.
        """
        self.dispatcher.dispatch(TripleAddedEvent(triple=(subject, predicate, object), context=context))

    def addN(self, quads):
       """
       Adds each item in the list of statements to a specific context. The quoted argument
       is interpreted by formula-aware stores to indicate this statement is quoted/hypothetical.
       Note that the default implementation is a redirect to add
       """
       for s,p,o,c in quads:
           assert c is not None, "Context associated with %s %s %s is None!"%(s,p,o)
           self.add(
                     (s,p,o),
                     c
            )

    def remove(self, (subject, predicate, object), context=None):
        """ Remove the set of triples matching the pattern from the store """
        self.dispatcher.dispatch(TripleRemovedEvent(triple=(subject, predicate, object), context=context))

    def triples_choices(self, (subject, predicate, object_),context=None):
        """
        A variant of triples that can take a list of terms instead of a single
        term in any slot.  Stores can implement this to optimize the response time
        from the default 'fallback' implementation, which will iterate
        over each term in the list and dispatch to tripless
        """
        if isinstance(object_,list):
            assert not isinstance(subject,list), "object_ / subject are both lists"
            assert not isinstance(predicate,list), "object_ / predicate are both lists"
            if object_:
                for obj in object_:
                    for (s1, p1, o1), cg in self.triples((subject,predicate,obj),context):
                        yield (s1, p1, o1), cg
            else:
                for (s1, p1, o1), cg in self.triples((subject,predicate,None),context):
                        yield (s1, p1, o1), cg

        elif isinstance(subject,list):
            assert not isinstance(predicate,list), "subject / predicate are both lists"
            if subject:
                for subj in subject:
                    for (s1, p1, o1), cg in self.triples((subj,predicate,object_),context):
                        yield (s1, p1, o1), cg
            else:
                for (s1, p1, o1), cg in self.triples((None,predicate,object_),context):
                    yield (s1, p1, o1), cg

        elif isinstance(predicate,list):
            assert not isinstance(subject,list), "predicate / subject are both lists"
            if predicate:
                for pred in predicate:
                    for (s1, p1, o1), cg in self.triples((subject,pred,object_),context):
                        yield (s1, p1, o1), cg
            else:
                for (s1, p1, o1), cg in self.triples((subject,None,object_),context):
                        yield (s1, p1, o1), cg

    def triples(self, (subject, predicate, object), context=None):
        """
        A generator over all the triples matching the pattern. Pattern can
        include any objects for used for comparing against nodes in the store, for
        example, REGEXTerm, URIRef, Literal, BNode, Variable, Graph, QuotedGraph, Date? DateRange?

        A conjunctive query can be indicated by either providing a value of None
        for the context or the identifier associated with the Conjunctive Graph (if it's context aware).
        """

    # variants of triples will be done if / when optimization is needed

    def __len__(self, context=None):
        """
        Number of statements in the store. This should only account for non-quoted (asserted) statements
        if the context is not specified, otherwise it should return the number of statements in the formula or context given.
        """

    def contexts(self, triple=None):
        """
        Generator over all contexts in the graph. If triple is specified, a generator over all
        contexts the triple is in.
        """

    # Optional Namespace methods

    def bind(self, prefix, namespace):
        """ """

    def prefix(self, namespace):
        """ """

    def namespace(self, prefix):
        """ """

    def namespaces(self):
        """ """
        if False:
            yield

    # Optional Transactional methods

    def commit(self):
        """ """

    def rollback(self):
        """ """

<commit_msg>Update issue 129 I have fixed the one "yield (nothing)" I could find. I no longer have python2.4 handy to make sure it works now.<commit_after>"""
============
rdflib.store
============

``Context-aware``: An RDF store capable of storing statements within contexts
is considered context-aware. Essentially, such a store is able to partition
the RDF model it represents into individual, named, and addressable
sub-graphs.

Relevant Notation3 reference regarding formulae, quoted statements, and such:
http://www.w3.org/DesignIssues/Notation3.html

``Formula-aware``: An RDF store capable of distinguishing between statements
that are asserted and statements that are quoted is considered formula-aware.

``Conjunctive Graph``: This refers to the 'top-level' Graph. It is the
aggregation of all the contexts within it and is also the appropriate,
absolute boundary for closed world assumptions / models.

For the sake of persistence, Conjunctive Graphs must be distinguished by
identifiers (that may not necessarily be RDF identifiers or may be an RDF
identifier normalized - SHA1/MD5 perhaps - for database naming purposes ).

``Conjunctive Query``: Any query that doesn't limit the store to search
within a named context only. Such a query expects a context-aware store to
search the entire asserted universe (the conjunctive graph). A formula-aware
store is expected not to include quoted statements when matching such a query.
"""

#Constants representing the state of a Store (returned by the open method)
VALID_STORE     = 1
CORRUPTED_STORE = 0
NO_STORE        = -1
UNKNOWN         = None

from rdflib.events import Dispatcher, Event

class StoreCreatedEvent(Event):
    """
    This event is fired when the Store is created, it has the folloing attribute:

      - 'configuration' string that is used to create the store

    """

class TripleAddedEvent(Event):
    """
    This event is fired when a triple is added, it has the following attributes:

      - 'triple' added to the graph
      - 'context' of the triple if any
      - 'graph' that the triple was added to
    """

class TripleRemovedEvent(Event):
    """
    This event is fired when a triple is removed, it has the following attributes:

      - 'triple' removed from the graph
      - 'context' of the triple if any
      - 'graph' that the triple was removed from
    """

from cPickle import Pickler, Unpickler, UnpicklingError
from cStringIO import StringIO

class NodePickler(object):
    def __init__(self):
        self._objects = {}
        self._ids = {}
        self._get_object = self._objects.__getitem__

    def _get_ids(self, key):
        try:
            return self._ids.get(key)
        except TypeError, e:
            return None

    def register(self, object, id):
        self._objects[id] = object
        self._ids[object] = id

    def loads(self, s):
        up = Unpickler(StringIO(s))
        up.persistent_load = self._get_object
        try:
            return up.load()
        except KeyError, e:
            raise UnpicklingError, "Could not find Node class for %s" % e

    def dumps(self, obj, protocol=None, bin=None):
        src = StringIO()
        p = Pickler(src)
        p.persistent_id = self._get_ids
        p.dump(obj)
        return src.getvalue()

class Store(object):
    #Properties
    context_aware = False
    formula_aware = False
    transaction_aware = False
    batch_unification = False
    def __init__(self, configuration=None, identifier=None):
        """
        identifier: URIRef of the Store. Defaults to CWD
        configuration: string containing infomation open can use to
        connect to datastore.
        """
        self.__node_pickler = None
        self.dispatcher = Dispatcher()
        if configuration:
            self.open(configuration)

    def __get_node_pickler(self):
        if self.__node_pickler is None:
            from rdflib.term import URIRef
            from rdflib.term import BNode
            from rdflib.term import Literal
            from rdflib.graph import Graph, QuotedGraph, GraphValue
            from rdflib.term import Variable
            from rdflib.term import Statement
            self.__node_pickler = np = NodePickler()
            np.register(self, "S")
            np.register(URIRef, "U")
            np.register(BNode, "B")
            np.register(Literal, "L")
            np.register(Graph, "G")
            np.register(QuotedGraph, "Q")
            np.register(Variable, "V")
            np.register(Statement, "s")
            np.register(GraphValue, "v")
        return self.__node_pickler
    node_pickler = property(__get_node_pickler)

    #Database management methods
    def create(self, configuration):
        self.dispatcher.dispatch(StoreCreatedEvent(configuration=configuration))

    def open(self, configuration, create=False):
        """
        Opens the store specified by the configuration string. If
        create is True a store will be created if it does not already
        exist. If create is False and a store does not already exist
        an exception is raised. An exception is also raised if a store
        exists, but there is insufficient permissions to open the
        store.  This should return one of VALID_STORE,CORRUPTED_STORE,or NO_STORE
        """
        return UNKNOWN

    def close(self, commit_pending_transaction=False):
        """
        This closes the database connection. The commit_pending_transaction parameter specifies whether to
        commit all pending transactions before closing (if the store is transactional).
        """

    def destroy(self, configuration):
        """
        This destroys the instance of the store identified by the configuration string.
        """

    def gc(self):
        """
        Allows the store to perform any needed garbage collection
        """
        pass

    #RDF APIs
    def add(self, (subject, predicate, object), context, quoted=False):
        """
        Adds the given statement to a specific context or to the model. The quoted argument
        is interpreted by formula-aware stores to indicate this statement is quoted/hypothetical
        It should be an error to not specify a context and have the quoted argument be True.
        It should also be an error for the quoted argument to be True when the store is not formula-aware.
        """
        self.dispatcher.dispatch(TripleAddedEvent(triple=(subject, predicate, object), context=context))

    def addN(self, quads):
       """
       Adds each item in the list of statements to a specific context. The quoted argument
       is interpreted by formula-aware stores to indicate this statement is quoted/hypothetical.
       Note that the default implementation is a redirect to add
       """
       for s,p,o,c in quads:
           assert c is not None, "Context associated with %s %s %s is None!"%(s,p,o)
           self.add(
                     (s,p,o),
                     c
            )

    def remove(self, (subject, predicate, object), context=None):
        """ Remove the set of triples matching the pattern from the store """
        self.dispatcher.dispatch(TripleRemovedEvent(triple=(subject, predicate, object), context=context))

    def triples_choices(self, (subject, predicate, object_),context=None):
        """
        A variant of triples that can take a list of terms instead of a single
        term in any slot.  Stores can implement this to optimize the response time
        from the default 'fallback' implementation, which will iterate
        over each term in the list and dispatch to tripless
        """
        if isinstance(object_,list):
            assert not isinstance(subject,list), "object_ / subject are both lists"
            assert not isinstance(predicate,list), "object_ / predicate are both lists"
            if object_:
                for obj in object_:
                    for (s1, p1, o1), cg in self.triples((subject,predicate,obj),context):
                        yield (s1, p1, o1), cg
            else:
                for (s1, p1, o1), cg in self.triples((subject,predicate,None),context):
                        yield (s1, p1, o1), cg

        elif isinstance(subject,list):
            assert not isinstance(predicate,list), "subject / predicate are both lists"
            if subject:
                for subj in subject:
                    for (s1, p1, o1), cg in self.triples((subj,predicate,object_),context):
                        yield (s1, p1, o1), cg
            else:
                for (s1, p1, o1), cg in self.triples((None,predicate,object_),context):
                    yield (s1, p1, o1), cg

        elif isinstance(predicate,list):
            assert not isinstance(subject,list), "predicate / subject are both lists"
            if predicate:
                for pred in predicate:
                    for (s1, p1, o1), cg in self.triples((subject,pred,object_),context):
                        yield (s1, p1, o1), cg
            else:
                for (s1, p1, o1), cg in self.triples((subject,None,object_),context):
                        yield (s1, p1, o1), cg

    def triples(self, (subject, predicate, object), context=None):
        """
        A generator over all the triples matching the pattern. Pattern can
        include any objects for used for comparing against nodes in the store, for
        example, REGEXTerm, URIRef, Literal, BNode, Variable, Graph, QuotedGraph, Date? DateRange?

        A conjunctive query can be indicated by either providing a value of None
        for the context or the identifier associated with the Conjunctive Graph (if it's context aware).
        """

    # variants of triples will be done if / when optimization is needed

    def __len__(self, context=None):
        """
        Number of statements in the store. This should only account for non-quoted (asserted) statements
        if the context is not specified, otherwise it should return the number of statements in the formula or context given.
        """

    def contexts(self, triple=None):
        """
        Generator over all contexts in the graph. If triple is specified, a generator over all
        contexts the triple is in.
        """

    # Optional Namespace methods

    def bind(self, prefix, namespace):
        """ """

    def prefix(self, namespace):
        """ """

    def namespace(self, prefix):
        """ """

    def namespaces(self):
        """ """
        if False:
            yield None

    # Optional Transactional methods

    def commit(self):
        """ """

    def rollback(self):
        """ """

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os, urllib2, requests, json

# Define the Rexster base URL
base_url = u"http://your_server:{port:d}/{path}"

def encode_non_ascii(s):
    return urllib2.quote(s.encode('utf8'),
                         ''.join([chr(i) for i in range(128)]))

def is_possible_wiki_slug(s):
    if s.islower():
        # User-generated slugs are lowercase
        if not s[0].isdigit():
            return False
        else:
            # Some wikislugs are lowercase but start with a digit
            return True
    else:
        # Most wikislugs contain a capital character
        return True

def get_category_map(topic_slug_list, language_code, port=8182):

    # Set up language-specific processing
    def slug_esc(s):
        return s

    if language_code == u"nl":
        prefix = u"dbp-nl:"
    else:
        prefix = u"dbp:"

        # English DBpedia uses URI instead of IRI, thus needs to escape non-ASCII
        def slug_esc(s):
            return encode_non_ascii(s)

    # Build the Gremlin script string
    prefixed_slugs = [u'"{0}{1}"'.format(prefix, slug_esc(slug)) for slug in topic_slug_list
                      if slug is not None and is_possible_wiki_slug(slug)]
    script_value = u"getCatMap([{0}], '{1}')".format(u", ".join(prefixed_slugs), language_code)

    url = base_url.format(port=port, path="graphs/dbp-sail/tp/gremlin")
    payload = {'script': script_value, 'load': "[v0_categories]"}

    print url, json.dumps(payload)

    resp = requests.post(url, data=payload)

    try:
        resp_dict = resp.json()
        if resp.status_code is not 200:
            print resp_dict
            return {"n_found_cats": 0}
        resp_dict['n_found_cats'] = len(resp_dict["results"][0])
        return resp_dict
    except Exception as e:
        print "Server error:", e
        return {"n_found_cats": 0}

def cache_related_resources(dir_path):
    ann_file_paths = [os.path.join(dir_path, fn) for fn
        in os.listdir(dir_path) if fn.endswith("_annotations.json")]

    for fp in ann_file_paths:
        slug_set = set()
        with open(fp, 'rb') as f:
            annotations = json.load(f)
        for ann in annotations:
            for cand in ann[u'resource']:
                slug_set.add(unicode(cand[u'uri']))
        cat_map = get_category_map(list(slug_set), 'en')
        print "N found cats:", cat_map['n_found_cats'], "\n"<commit_msg>Included get_flow_map calls, because get_category_map only dereferences categories multiple levels out. The desired behavior is to dereference all neighboring resources.<commit_after># -*- coding: utf-8 -*-
import os, urllib2, requests, json

# Define the Rexster base URL
base_url = u"http://your_server:{port:d}/{path}"

def encode_non_ascii(s):
    return urllib2.quote(s.encode('utf8'),
                         ''.join([chr(i) for i in range(128)]))

def is_possible_wiki_slug(s):
    if s.islower():
        # User-generated slugs are lowercase
        if not s[0].isdigit():
            return False
        else:
            # Some wikislugs are lowercase but start with a digit
            return True
    else:
        # Most wikislugs contain a capital character
        return True

def get_category_map(topic_slug_list, language_code, port=8182):

    # Set up language-specific processing
    def slug_esc(s):
        return s

    if language_code == u"nl":
        prefix = u"dbp-nl:"
    else:
        prefix = u"dbp:"

        # English DBpedia uses URI instead of IRI, thus needs to escape non-ASCII
        def slug_esc(s):
            return encode_non_ascii(s)

    # Build the Gremlin script string
    prefixed_slugs = [u'"{0}{1}"'.format(prefix, slug_esc(slug)) for slug in topic_slug_list
                      if slug is not None and is_possible_wiki_slug(slug)]
    script_value = u"getCatMap([{0}], '{1}')".format(u", ".join(prefixed_slugs), language_code)

    url = base_url.format(port=port, path="graphs/dbp-sail/tp/gremlin")
    payload = {'script': script_value, 'load': "[v0_categories]"}

    print url, json.dumps(payload)

    resp = requests.post(url, data=payload)

    try:
        resp_dict = resp.json()
        if resp.status_code is not 200:
            print resp_dict
            return {"n_found_cats": 0}
        resp_dict['n_found_cats'] = len(resp_dict["results"][0])
        return resp_dict
    except Exception as e:
        print "Server error:", e
        return {"n_found_cats": 0}

def get_flow_map(topic_slug_list, language_code, port=8182):
    # Build the Gremlin script string
    if language_code == u'en':
        escaped_slugs = [u'"{0}"'.format(encode_non_ascii(slug)) for slug in topic_slug_list
                         if slug is not None and is_possible_wiki_slug(slug)]
    else:
        escaped_slugs = [u'"{0}"'.format(slug) for slug in topic_slug_list
                         if slug is not None and is_possible_wiki_slug(slug)]
    script_value = u"getFlowMap([{0}], '{1}')".format(u", ".join(escaped_slugs), language_code)

    url = base_url.format(port=port, path="graphs/dbp-sail/tp/gremlin")
    payload = {'script': script_value, 'load': "[v0_categories]"}

    print url, json.dumps(payload)

    resp = requests.post(url, data=payload)

    try:
        resp_dict = resp.json()
        if resp.status_code is not 200:
            print resp_dict
            return {"n_related_topics": 0}
        resp_dict['n_related_topics'] = len(resp_dict["results"][0])
        return resp_dict
    except Exception as e:
        print "Server error:", e
        return {"n_related_topics": 0}

def cache_related_resources(dir_path):
    ann_file_paths = [os.path.join(dir_path, fn) for fn
        in os.listdir(dir_path) if fn.endswith("_annotations.json")]

    for fp in ann_file_paths:
        slug_set = set()
        with open(fp, 'rb') as f:
            annotations = json.load(f)
        for ann in annotations:
            for cand in ann[u'resource']:
                slug_set.add(unicode(cand[u'uri']))
        cat_map = get_category_map(list(slug_set), 'en')
        print "N found cats:", cat_map['n_found_cats'], "\n"
        flow_map = get_flow_map(list(slug_set), 'en')
        print "N related resources:", flow_map['n_related_topics'], "\n"<|endoftext|>


<commit_before>#! /usr/bin/env python
#
#  Load a FITS cube , extract the spectrum at a (or reference) pixel
#  and operate and plot some and then more....
#
#
#  22-jun-2017  PJT             summer project - cloned off cubespectrum.py
#  july-2017    Thomas/Peter    various improvements
#
#  @todo
#     - have optional RESTFRQ or RESTFREQ as 3rd argument [done]
#     - output the spectrum in a table, much like testCubeSpectrum.tab [done]
#     - resample the gauss finer (not 5 points but may be 10x more?)

import os, sys, math
import numpy as np
import numpy.ma as ma
import matplotlib.pyplot as plt
import pyspeckit

from astropy.io import fits
from astropy.units import Quantity
c = 299792.458     # [km/s] there should be a way to get 'c' from astropy.units ?

na = len(sys.argv)
if na == 7:
# Must be in Km/s
    fitsfile = sys.argv[1]
    pos = [int(sys.argv[2]),int(sys.argv[3])]
    restfreq = float(sys.argv[4])* 1e9
    vmin = float(sys.argv[5])
    vmax = float(sys.argv[6])
    use_vel = True
elif na == 5:
# Must be in GHz
    fitsfile = sys.argv[1]
    pos = [int(sys.argv[2]),int(sys.argv[3])]
    vmin = vmax = None
    restfreq = float(sys.argv[4])* 1e9
    use_vel = True
elif na == 4:
# Pixel position
    fitsfile = sys.argv[1]
    pos = [int(sys.argv[2]),int(sys.argv[3])]
    restfreq = None
    vmin = vmax = None
    use_vel = False
elif na == 2:
# Fits file
    fitsfile = sys.argv[1]
    pos = None
    restfreq = None
    vmin = vmax = None
    use_vel = False
else:
    sys.exit(1)

# open the fits file
hdu = fits.open(fitsfile)
print(len(hdu))

# get a reference to the header and data.  Data should be 3dim numpy array now
h = hdu[0].header
d = hdu[0].data.squeeze()
print(d.shape)

#  grab the restfreq, there are at least two ways how this is done
if restfreq == None:
    if 'RESTFRQ' in h:
        restfreq=h['RESTFRQ']
    elif 'RESTFREQ' in h:
        restfreq=h['RESTFREQ']
    else:
        restfreq= h['CRVAL3']
print("RESTFREQ",restfreq)

if pos == None:
    # the FITS reference pixel is always a good backup
    xpos = int(h['CRPIX1'])
    ypos = int(h['CRPIX2'])
    print("No position given, using reference pixel %g %g" % (xpos,ypos))
else:
    xpos = pos[0]
    ypos = pos[1]

flux     = d[:,ypos,xpos]

nchan    = d.shape[0]
channeln = np.arange(nchan)
zero     = np.zeros(nchan)

cdelt3 = h['CDELT3']
crval3 = h['CRVAL3']
crpix3 = h['CRPIX3']
# to convert the channel to frequency
channelf = (channeln-crpix3+1)*cdelt3 + crval3
# to convert the Frequency to velocity
#channelv = (1.0-channelf/restfreq) * c
#print (channelf)
#print (channelv)

# what we plot
#channel = channelv
#channel = channelf
#channel = channeln

if use_vel:
# to convert the Frequency to velocity
    channelv = (1.0-channelf/restfreq) * c
    channel = channelv
    print (channelv.min())
    print (channelv.max())
else:
    channel = channelf
print (channelf.min()/1e9)
print (channelf.max()/1e9)

def gfit1(xi,yi,m=5):
    """
    moments around a peak
    (also) rely on number of pixels left and right of the peak. Masking optional
    """
    print("GFIT1")
    ipeak = yi.argmax()
    xpeak = xi[ipeak]
    ypeak = yi[ipeak]
    print(ipeak,xpeak,ypeak)
    print(xi[ipeak-m:ipeak+m])
    print(yi[ipeak-m:ipeak+m])
    x = xi[ipeak-m:ipeak+m]
    y = yi[ipeak-m:ipeak+m]
    print(xi.shape)
    print(yi.shape)
    xmean = (x*y).sum() / y.sum()
    xdisp = (x*x*y).sum() / y.sum() - xmean*xmean
    if xdisp > 0:
        xdisp = math.sqrt(xdisp)
    fwhm = 2.355 * xdisp
    print("MEAN/DISP/FWHM:",xmean,xdisp,fwhm)
    ymodel = ypeak * np.exp(-0.5*(xi-xmean)**2/(xdisp*xdisp))
    return ymodel

def gfit2(x,y):
    """
    rely on masking completely
    moments around a peak
    """
    print("GFIT2")
    xmean = (x*y).sum() / y.sum()
    xdisp = (x*x*y).sum() / y.sum() - xmean*xmean
    if xdisp > 0:
        xdisp = math.sqrt(xdisp)
    fwhm = 2.355 * xdisp
    print("MEAN/DISP/FWHM:",xmean,xdisp,fwhm)
    ypeak = y.max()
    print(ypeak)
    ymodel = ypeak * np.exp(-0.5*(x-xmean)**2/(xdisp*xdisp))
    return ymodel

def gfit3(xi,yi):
    """
    relies on masking , use pyspeckit
    """
    # not sure if we need this, or try x = xi
    x = ma.compressed(xi)
    y = ma.compressed(yi)
    sp = pyspeckit.Spectrum(data=y, xarr=x, error=None, header=h,)
    sp.plotter()
    sp.specfit(fittype='gaussian')
    sp.specfit.plot_fit()
#    sp.baseline()
    print(x)
    print(y)
    # fake a return array
    return yi

def gfit4(x,y):
    """
        relies on masking , use scipy's curve_fit
    """
    def gauss(x, *p):
        A, mu, sigma, B = p
        return A*np.exp(-(x-mu)**2/(2.*sigma**2)) + B
    # the curve fit stuff will go here
    #
    return y

if use_vel == True:
   plt.figure()
   if vmin != None:
       channelv = ma.masked_outside(channelv,vmin,vmax)
       flux     = ma.masked_array(flux, channelv.mask)
       plt.xlim([vmin,vmax])
       if True:
           ymodel = gfit3(channelv,flux)
           plt.plot(channelv,ymodel,label='gfit1')

   plt.plot(channelv,flux,'o-',markersize=2,label='data')
   plt.plot(channelv,zero)
   plt.xlabel("Velocity (km/s)")
   plt.ylabel("Flux")
   plt.title(fitsfile +"  @ %g %g" % (xpos,ypos)+ "   %g" % (restfreq/1e9)+ 'Ghz')
   plt.legend()
   plt.show()
else:
   plt.figure()
   plt.plot(channelf/1e9,flux,'o-',markersize=2,label='data')
   plt.plot(channelf/1e9,zero)
   plt.xlabel("Frequency (GHz)")
   plt.ylabel("Flux")
   plt.title(fitsfile + " @ %g %g" % (xpos,ypos))
   plt.legend()
   plt.show()

#to create a table of the frequency and flux
xtab = channelf /1e9 #to set the freqency to GHz
ytab = flux
np.savetxt('Frequency_Flux.tab',np.c_[xtab,ytab], delimiter='  ',header=("Frequency""       " "Flux"),comments='#',fmt='%.8f')

<commit_msg>add 8th op[tional argument to fit in sky freq<commit_after>#! /usr/bin/env python
#
#  Load a FITS cube , extract the spectrum at a (or reference) pixel
#  and operate and plot some and then more....
#
#
#  22-jun-2017  PJT             summer project - cloned off cubespectrum.py
#  july-2017    Thomas/Peter    various improvements
#
#  @todo
#     - have optional RESTFRQ or RESTFREQ as 3rd argument [done]
#     - output the spectrum in a table, much like testCubeSpectrum.tab [done]
#     - resample the gauss finer (not 5 points but may be 10x more?)

import os, sys, math
import numpy as np
import numpy.ma as ma
import matplotlib.pyplot as plt
import pyspeckit
from scipy.optimize import curve_fit

from astropy.io import fits
from astropy.units import Quantity
c = 299792.458     # [km/s] there should be a way to get 'c' from astropy.units ?

#  set common command line options (unless overriden below)
vlsr = None

na = len(sys.argv)
if na == 8:
    fitsfile = sys.argv[1]
    pos = [int(sys.argv[2]),int(sys.argv[3])]
    restfreq = float(sys.argv[4])* 1e9
    vmin = float(sys.argv[5])
    vmax = float(sys.argv[6])
    vlsr = float(sys.argv[7])
    use_vel = False
elif na == 7:
    # Must be in Km/s
    fitsfile = sys.argv[1]
    pos = [int(sys.argv[2]),int(sys.argv[3])]
    restfreq = float(sys.argv[4])* 1e9
    vmin = float(sys.argv[5])
    vmax = float(sys.argv[6])
    use_vel = True
elif na == 5:
    # Must be in GHz
    fitsfile = sys.argv[1]
    pos = [int(sys.argv[2]),int(sys.argv[3])]
    vmin = vmax = None
    restfreq = float(sys.argv[4])* 1e9
    use_vel = True
elif na == 4:
    # Pixel position
    fitsfile = sys.argv[1]
    pos = [int(sys.argv[2]),int(sys.argv[3])]
    restfreq = None
    vmin = vmax = None
    use_vel = False
elif na == 2:
    # Fits file
    fitsfile = sys.argv[1]
    pos = None
    restfreq = None
    vmin = vmax = None
    use_vel = False
else:
    print("Usage: %s fitsfile [xpos ypos] [restfreq [vmin vmax] [vlsr]" % sys.argv[0])
    sys.exit(1)

# open the fits file
hdu = fits.open(fitsfile)
print(len(hdu))

# get a reference to the header and data.  Data should be 3dim numpy array now
h = hdu[0].header
d = hdu[0].data.squeeze()
print(d.shape)

#  grab the restfreq, there are at least two ways how this is done
if restfreq == None:
    if 'RESTFRQ' in h:
        restfreq=h['RESTFRQ']
    elif 'RESTFREQ' in h:
        restfreq=h['RESTFREQ']
    else:
        restfreq= h['CRVAL3']
print("RESTFREQ",restfreq/1e9)

if pos == None:
    # the FITS reference pixel is always a good backup
    xpos = int(h['CRPIX1'])
    ypos = int(h['CRPIX2'])
    print("No position given, using reference pixel %g %g" % (xpos,ypos))
else:
    xpos = pos[0]
    ypos = pos[1]

flux     = d[:,ypos,xpos]

nchan    = d.shape[0]
channeln = np.arange(nchan)
zero     = np.zeros(nchan)

cdelt3 = h['CDELT3']
crval3 = h['CRVAL3']
crpix3 = h['CRPIX3']
# to convert the channel to frequency
channelf = (channeln-crpix3+1)*cdelt3 + crval3
# to convert the Frequency to velocity
#channelv = (1.0-channelf/restfreq) * c
#print (channelf)
#print (channelv)

# what we plot
#channel = channelv
#channel = channelf
#channel = channeln

# to convert the Frequency to velocity
channelv = (1.0-channelf/restfreq) * c
print (channelv.min(), channelv.max())
print (channelf.min()/1e9, channelf.max()/1e9)

def gfit1(xi,yi,m=5):
    """
    moments around a peak
    (also) rely on number of pixels left and right of the peak. Masking optional
    """
    print("GFIT1")
    ipeak = yi.argmax()
    xpeak = xi[ipeak]
    ypeak = yi[ipeak]
    print(ipeak,xpeak,ypeak)
    print(xi[ipeak-m:ipeak+m])
    print(yi[ipeak-m:ipeak+m])
    x = xi[ipeak-m:ipeak+m]
    y = yi[ipeak-m:ipeak+m]
    print(xi.shape)
    print(yi.shape)
    xmean = (x*y).sum() / y.sum()
    xdisp = (x*x*y).sum() / y.sum() - xmean*xmean
    if xdisp > 0:
        xdisp = math.sqrt(xdisp)
    fwhm = 2.355 * xdisp
    print("MEAN/DISP/FWHM:",xmean,xdisp,fwhm)
    ymodel = ypeak * np.exp(-0.5*(xi-xmean)**2/(xdisp*xdisp))
    return ymodel

def gfit2(x,y):
    """
    rely on masking completely
    moments around a peak
    """
    print("GFIT2")
    xmean = (x*y).sum() / y.sum()
    xdisp = (x*x*y).sum() / y.sum() - xmean*xmean
    if xdisp > 0:
        xdisp = math.sqrt(xdisp)
    fwhm = 2.355 * xdisp
    print("MEAN/DISP/FWHM:",xmean,xdisp,fwhm)
    ypeak = y.max()
    print(ypeak)
    ymodel = ypeak * np.exp(-0.5*(x-xmean)**2/(xdisp*xdisp))
    return ymodel

def gfit3(xi,yi):
    """
    relies on masking , use pyspeckit
    """
    # not sure if we need this, or try x = xi
    x = ma.compressed(xi)
    y = ma.compressed(yi)
    sp = pyspeckit.Spectrum(data=y, xarr=x, error=None, header=h,)
    sp.plotter()
    sp.specfit(fittype='gaussian')
    sp.specfit.plot_fit()
    # sp.baseline()
    print(x)
    print(y)
    # fake a return array
    return yi

def gfit4(x,y):
    """
        relies on masking , use scipy's curve_fit
    """
    def gauss(x, *p):
        A, mu, sigma, B = p
        return A*np.exp(-(x-mu)**2/(2.*sigma**2)) + B
    #
    # first get some initial estimates
    B = y.min()
    A = y.max() - B
    sigma = (x.max() - x.min() ) /2.0 / 2.5     # this can be done better, moment analysis?
    mu    = (x.max() + x.min() ) /2.0
    p0 = [A, mu, sigma, B]
    # p0 = [0.030544054, 50, 200, 0.029194169]

    print("p0 = ",p0)

    coeff, cm = curve_fit(gauss, x, y, p0=p0)
    ymodel = gauss(x, *coeff)

    print("Fitted amp            :",coeff[0])
    print("Fitted mean           :",coeff[1])
    print("Fitted sigma and FWHM :",coeff[2], coeff[2]*2.355)
    print("Fitted baseline       :",coeff[3])
    print("Covariance Matrix     :\n",cm)
    # what are now the errors in the fitted values?
    print("error amp     :",math.sqrt(cm[0][0]))
    print("error mean    :",math.sqrt(cm[1][1]))
    print("error sigma   :",math.sqrt(cm[2][2]))
    print("error baseline:",math.sqrt(cm[3][3]))

    return ymodel,coeff[1]

if use_vel == True:
    plt.figure()
    if vmin != None:
        # mask
        channelv = ma.masked_outside(channelv,vmin,vmax)
        flux     = ma.masked_array(flux, channelv.mask)
        # make arrays smaller
        channelv = ma.compressed(channelv)
        flux     = ma.compressed(flux)
        # plotting
        # plt.xlim([vmin,vmax])         # technically not needed
        ymodel,vfit = gfit4(channelv,flux)
        plt.plot(channelv,ymodel,label='gfit4')

    plt.plot(channelv,flux,'o-',markersize=2,label='data')
    # plt.plot(channelv,zero)
    plt.xlabel("Velocity (km/s)")
    plt.ylabel("Flux")
    plt.title(fitsfile +"  @ %g %g" % (xpos,ypos)+ "   %g" % (restfreq/1e9)+ 'Ghz')
    plt.legend()
    plt.show()
else:
    plt.figure()
    if vlsr != None:
        print ("TODO")
        channelv = ma.masked_outside(channelv,vmin,vmax)
        channelf = ma.masked_array(channelf, channelv.mask)
        flux     = ma.masked_array(flux,     channelv.mask)
        #
        channelv = ma.compressed(channelv)
        channelf = ma.compressed(channelf)
        flux     = ma.compressed(flux)

        ymodel,ffit = gfit4(channelf/1e9,flux)
        plt.plot(channelf/1e9,ymodel,label='gfit4')
        plt.plot(channelf/1e9,flux,'o-',markersize=2,label='data')

        # compute restfreq
        f0 = ffit / (1-vlsr/c)
        print("Fitted restfreq f0=",f0)
    else:
        plt.plot(channelf/1e9,flux,'o-',markersize=2,label='data')
        plt.plot(channelf/1e9,zero)
    plt.xlabel("Frequency (GHz)")
    plt.ylabel("Flux")
    plt.title(fitsfile + " @ %g %g" % (xpos,ypos))
    plt.legend()
    plt.show()

#to create a table of the frequency and flux
xtab = channelf /1e9 #to set the freqency to GHz
ytab = flux
np.savetxt('Frequency_Flux.tab',np.c_[xtab,ytab], delimiter='  ',header=("Frequency""       " "Flux"),comments='#',fmt='%.8f')

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
###############################################################################
#
# ODOO (ex OpenERP)
# Open Source Management Solution
# Copyright (C) 2001-2015 Micronaet S.r.l. (<https://micronaet.com>)
# Developer: Nicola Riolini @thebrush (<https://it.linkedin.com/in/thebrush>)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class ProductProduct(orm.Model):
    """ Model name: ProductProduct add utility for report
    """
    _inherit = 'product.product'

    def report_get_objects_bom_industrial_cost(self, cr, uid, datas=None,
            context=None):
        ''' Integration report
        '''
        _logger.info('Start extracting...')
        res = super(
            ProductProduct, self).report_get_objects_bom_industrial_cost(
                cr, uid, datas=context, context=context)
        _logger.info('Start reporting...')

        # ---------------------------------------------------------------------
        # NEW VERSIONE:
        # ---------------------------------------------------------------------
        data = {
            'wizard': True,
            'active_ids': False, # Load all template
            }

        res = product_pool.report_get_objects_bom_industrial_cost(
            cr, uid, data=data)

        f_bom = open('/home/odoo/0.DB.bom_file.csv', 'w')

        header = 'Componente|Q.|UM|Prezzo|Totale\n'
        for item in res:
            (min, max, error, components, extra1, extra2, index, total,
                product, parameter, total_text, pipe_total_weight) = item
            f_bom('%s|%s' % (
                product.default_code,
                product.name,
                ))

            # -----------------------------------------------------------------
            # Component line:
            # -----------------------------------------------------------------
            f_bom.write(header)
            for line in components:
                (name, q, um, price, total, list_detail, hw, element,
                    red_price, fabric_text) = line
                if test == 'hw':
                    block1 = '%s%s' % (
                        name,
                        list_ids,
                        hw.default_code,
                        )
                else:
                    block1 = name

                f_bom.write('%s%s|%s|%s|%s|%s\n' % (
                    block1,
                    q,
                    um,
                    price,
                    total,
                    ))

            # -----------------------------------------------------------------
            # Work
            # -----------------------------------------------------------------
            f_bom.write('Manodopera||Prodotto|Totale')
            for item, details, time_qty in extra1:
                f_bom.write('%s (unit. %s)||%s|%s\n' % (
                    item.cost_id.name,
                    item.cost_id.unit_cost,
                    item.product_id.default_code or '/',
                    details,
                    ))

            # -----------------------------------------------------------------
            # Industrial
            # -----------------------------------------------------------------
            f_bom.write('Costi industriali||Prodotto|Totale')
            for item, details, time_qty in extra2:
                f_bom.write('%s (unit. %s)||%s|%s\n' % (
                    item.cost_id.name,
                    item.cost_id.unit_cost,
                    item.product_id.default_code or '/',
                    details,
                    ))

            # -----------------------------------------------------------------
            # Total:
            # -----------------------------------------------------------------
            f_bom.write('Subtotali|%s|Costo|%s-%s\n' % (
                total_test, min, max
                ))
            #f_bom.write('Indici|%s|Vendita|%s-%s\n' % (
            #    total['index'],
            #    total['margin']
            #    ))
        return res

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>generic bugfix (fast message)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
###############################################################################
#
# ODOO (ex OpenERP)
# Open Source Management Solution
# Copyright (C) 2001-2015 Micronaet S.r.l. (<https://micronaet.com>)
# Developer: Nicola Riolini @thebrush (<https://it.linkedin.com/in/thebrush>)
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID, api
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class ProductProduct(orm.Model):
    """ Model name: ProductProduct add utility for report
    """
    _inherit = 'product.product'

    def report_get_objects_bom_industrial_cost(self, cr, uid, datas=None,
            context=None):
        ''' Integration report
        '''
        import pdb; pdb.set_trace()
        _logger.info('Start extracting...')
        res = super(
            ProductProduct, self).report_get_objects_bom_industrial_cost(
                cr, uid, datas=context, context=context)
        _logger.info('Start reporting...')

        # ---------------------------------------------------------------------
        # NEW VERSIONE:
        # ---------------------------------------------------------------------
        data = {
            'wizard': True,
            'active_ids': False, # Load all template
            }

        res = product_pool.report_get_objects_bom_industrial_cost(
            cr, uid, data=data)

        f_bom = open('/home/odoo/0.DB.bom_file.csv', 'w')

        header = 'Componente|Q.|UM|Prezzo|Totale\n'
        for item in res:
            (min, max, error, components, extra1, extra2, index, total,
                product, parameter, total_text, pipe_total_weight) = item
            f_bom('%s|%s' % (
                product.default_code,
                product.name,
                ))

            # -----------------------------------------------------------------
            # Component line:
            # -----------------------------------------------------------------
            f_bom.write(header)
            for line in components:
                (name, q, um, price, total, list_detail, hw, element,
                    red_price, fabric_text) = line
                if test == 'hw':
                    block1 = '%s%s' % (
                        name,
                        list_ids,
                        hw.default_code,
                        )
                else:
                    block1 = name

                f_bom.write('%s%s|%s|%s|%s|%s\n' % (
                    block1,
                    q,
                    um,
                    price,
                    total,
                    ))

            # -----------------------------------------------------------------
            # Work
            # -----------------------------------------------------------------
            f_bom.write('Manodopera||Prodotto|Totale')
            for item, details, time_qty in extra1:
                f_bom.write('%s (unit. %s)||%s|%s\n' % (
                    item.cost_id.name,
                    item.cost_id.unit_cost,
                    item.product_id.default_code or '/',
                    details,
                    ))

            # -----------------------------------------------------------------
            # Industrial
            # -----------------------------------------------------------------
            f_bom.write('Costi industriali||Prodotto|Totale')
            for item, details, time_qty in extra2:
                f_bom.write('%s (unit. %s)||%s|%s\n' % (
                    item.cost_id.name,
                    item.cost_id.unit_cost,
                    item.product_id.default_code or '/',
                    details,
                    ))

            # -----------------------------------------------------------------
            # Total:
            # -----------------------------------------------------------------
            f_bom.write('Subtotali|%s|Costo|%s-%s\n' % (
                total_test, min, max
                ))
            #f_bom.write('Indici|%s|Vendita|%s-%s\n' % (
            #    total['index'],
            #    total['margin']
            #    ))
        return res

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>


<commit_before>from pytest import raises

bad_response = {
    'base': {
        'ref': 'master'
    }
}

good_response = {
    'base': {
        'ref': 'not_master'
    }
}

def check_base_branch_test(requests_mock):
    from Tests.scripts.check_base_branch_for_external_pr import check_base_branch
    url = 'https://api.github.com/repos/demisto/content/pulls/528'
    requests_mock.get(url, json=bad_response)
    with raises(SystemExit) as se:
        check_base_branch(528)
        assert se.value.code == 1
    requests_mock.get(url, json=good_response)
    with raises(SystemExit) as se:
        check_base_branch(528)
        assert se.value.code == 0<commit_msg>added line in eof<commit_after>from pytest import raises

bad_response = {
    'base': {
        'ref': 'master'
    }
}

good_response = {
    'base': {
        'ref': 'not_master'
    }
}

def check_base_branch_test(requests_mock):
    from Tests.scripts.check_base_branch_for_external_pr import check_base_branch
    url = 'https://api.github.com/repos/demisto/content/pulls/528'
    requests_mock.get(url, json=bad_response)
    with raises(SystemExit) as se:
        check_base_branch(528)
        assert se.value.code == 1
    requests_mock.get(url, json=good_response)
    with raises(SystemExit) as se:
        check_base_branch(528)
        assert se.value.code == 0
<|endoftext|>


<commit_before>import logging

__author__ = 'djw'

class Game(object):
    """
    Responsible for managing the flow of a game and all of its entities.
    """
    def __init__(self):
        # set up game parameters and initialize data structures
        logging.debug('initializing')

    # The current round of play
    current_round = 1

    # The number of rounds played before the game ends
    GAME_LENGTH = 15

    def run(self):
        # execute the iterative flow of the game
        while self.current_round < 15:
            print 'Round: %s' % self.current_round
            self.current_round += 1
        self.end()

    def end(self):
        # calculate score, print results
        score = self.calculate_final_score()
        print 'Your score was: %s' % score

    def calculate_final_score(self):
        # calculate the player's final score
        logging.debug('calculating final score for player')
        score = 0  # get stuff here
        return score

# start everything!
if __name__ == "__main__":
    game = Game()
    game.run()<commit_msg>fix constants<commit_after>import logging

__author__ = 'djw'

class Game(object):
    """
    Responsible for managing the flow of a game and all of its entities.
    """
    def __init__(self):
        # set up game parameters and initialize data structures
        logging.debug('initializing')

    # The current round of play
    current_round = 1

    # The number of rounds played before the game ends
    GAME_LENGTH = 14

    def run(self):
        # execute the iterative flow of the game
        while self.current_round <= self.GAME_LENGTH:
            print 'Round: %s' % self.current_round
            self.current_round += 1
        self.end()

    def end(self):
        # calculate score, print results
        score = self.calculate_final_score()
        print 'Your score was: %s' % score

    def calculate_final_score(self):
        # calculate the player's final score
        logging.debug('calculating final score for player')
        score = 0  # get stuff here
        return score

# start everything!
if __name__ == "__main__":
    game = Game()
    game.run()<|endoftext|>


<commit_before># All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""List and compare most used OpenStack cloud resources."""

import argparse
import json
import subprocess
import sys

from rally.common.plugin import discover
from rally import consts
from rally import osclients

class ResourceManager(object):

    REQUIRED_SERVICE = None
    REPR_KEYS = ("id", "name", "tenant_id", "zone", "zoneName", "pool")

    def __init__(self, clients):
        self.clients = clients

    def is_available(self):
        if self.REQUIRED_SERVICE:
            return self.REQUIRED_SERVICE in self.clients.services().values()
        return True

    @property
    def client(self):
        return getattr(self.clients, self.__class__.__name__.lower())()

    def get_resources(self):
        all_resources = []
        cls = self.__class__.__name__.lower()
        for prop in dir(self):
            if not prop.startswith("list_"):
                continue
            f = getattr(self, prop)
            resources = f() or []
            resource_name = prop[5:][:-1]

            for res in resources:
                res_repr = []
                for key in self.REPR_KEYS + (resource_name,):
                    if isinstance(res, dict):
                        value = res.get(key)
                    else:
                        value = getattr(res, key, None)
                    if value:
                        res_repr.append("%s:%s" % (key, value))
                if not res_repr:
                    raise ValueError("Failed to represent resource %r" % res)

                all_resources.append(
                    "%s %s %s" % (cls, resource_name, " ".join(res_repr)))
        return all_resources

class Keystone(ResourceManager):

    def list_users(self):
        return self.client.users.list()

    def list_tenants(self):
        return self.client.tenants.list()

    def list_roles(self):
        return self.client.roles.list()

class Nova(ResourceManager):

    def list_flavors(self):
        return self.client.flavors.list()

    def list_floating_ip_pools(self):
        return self.client.floating_ip_pools.list()

    def list_floating_ips(self):
        return self.client.floating_ips.list()

    def list_images(self):
        return self.client.images.list()

    def list_keypairs(self):
        return self.client.keypairs.list()

    def list_networks(self):
        return self.client.networks.list()

    def list_security_groups(self):
        return self.client.security_groups.list(
            search_opts={"all_tenants": True})

    def list_servers(self):
        return self.client.servers.list(
            search_opts={"all_tenants": True})

    def list_services(self):
        return self.client.services.list()

    def list_availability_zones(self):
        return self.client.availability_zones.list()

class Neutron(ResourceManager):

    REQUIRED_SERVICE = consts.Service.NEUTRON

    def has_extension(self, name):
        extensions = self.client.list_extensions().get("extensions", [])
        return any(ext.get("alias") == name for ext in extensions)

    def list_networks(self):
        return self.client.list_networks()["networks"]

    def list_subnets(self):
        return self.client.list_subnets()["subnets"]

    def list_routers(self):
        return self.client.list_routers()["routers"]

    def list_ports(self):
        return self.client.list_ports()["ports"]

    def list_floatingips(self):
        return self.client.list_floatingips()["floatingips"]

    def list_security_groups(self):
        return self.client.list_security_groups()["security_groups"]

    def list_health_monitors(self):
        if self.has_extension("lbaas"):
            return self.client.list_health_monitors()["health_monitors"]

    def list_pools(self):
        if self.has_extension("lbaas"):
            return self.client.list_pools()["pools"]

    def list_vips(self):
        if self.has_extension("lbaas"):
            return self.client.list_vips()["vips"]

class Glance(ResourceManager):

    def list_images(self):
        return self.client.images.list()

class Heat(ResourceManager):

    def list_resource_types(self):
        return self.client.resource_types.list()

    def list_stacks(self):
        return self.client.stacks.list()

class Cinder(ResourceManager):

    def list_availability_zones(self):
        return self.client.availability_zones.list()

    def list_backups(self):
        return self.client.backups.list()

    def list_volume_snapshots(self):
        return self.client.volume_snapshots.list()

    def list_volume_types(self):
        return self.client.volume_types.list()

    def list_volumes(self):
        return self.client.volumes.list(
            search_opts={"all_tenants": True})

class CloudResources(object):
    """List and compare cloud resources.

    resources = CloudResources(auth_url=..., ...)
    saved_list = resources.list()

    # Do something with the cloud ...

    changes = resources.compare(saved_list)
    has_changed = any(changes)
    removed, added = changes
    """

    def __init__(self, **kwargs):
        endpoint = osclients.objects.Endpoint(**kwargs)
        self.clients = osclients.Clients(endpoint)

    def _deduplicate(self, lst):
        """Change list duplicates to make all items unique.

        >>> resources._deduplicate(["a", "b", "c", "b", "b"])
        >>> ['a', 'b', 'c', 'b (duplicate 1)', 'b (duplicate 2)'
        """
        deduplicated_list = []
        for value in lst:
            if value in deduplicated_list:
                ctr = 0
                try_value = value
                while try_value in deduplicated_list:
                    ctr += 1
                    try_value = "%s (duplicate %i)" % (value, ctr)
                value = try_value
            deduplicated_list.append(value)
        return deduplicated_list

    def list(self):
        managers_classes = discover.itersubclasses(ResourceManager)
        resources = []
        for cls in managers_classes:
            manager = cls(self.clients)
            if manager.is_available():
                resources.extend(manager.get_resources())
        return sorted(self._deduplicate(resources))

    def compare(self, with_list):
        saved_resources = set(with_list)
        current_resources = set(self.list())
        removed = saved_resources - current_resources
        added = current_resources - saved_resources

        return sorted(list(removed)), sorted(list(added))

def main():

    parser = argparse.ArgumentParser(
        description=("Save list of OpenStack cloud resources or compare "
                     "with previously saved list."))
    parser.add_argument("--credentials",
                        type=argparse.FileType("r"),
                        metavar="<path/to/credentials.json>",
                        help="cloud credentials in JSON format")
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("--dump-list",
                       type=argparse.FileType("w"),
                       metavar="<path/to/output/list.json>",
                       help="dump resources to given file in JSON format")
    group.add_argument("--compare-with-list",
                       type=argparse.FileType("r"),
                       metavar="<path/to/existent/list.json>",
                       help=("compare current resources with a list from "
                             "given JSON file"))
    args = parser.parse_args()

    if args.credentials:
        config = json.load(args.credentials)
    else:
        config = json.loads(subprocess.check_output(["rally", "deployment",
                                                     "config"]))
        config.update(config.pop("admin"))
        del config["type"]

    resources = CloudResources(**config)

    if args.dump_list:
        resources_list = resources.list()
        json.dump(resources_list, args.dump_list, indent=2)
    elif args.compare_with_list:
        given_list = json.load(args.compare_with_list)
        changes = resources.compare(with_list=given_list)
        removed, added = changes
        sys.stdout.write(
            json.dumps({"removed": removed, "added": added}, indent=2))
        if any(changes):
            return 0  # `1' will fail gate job
    return 0

if __name__ == "__main__":
    sys.exit(main())
<commit_msg>[Ci] Check Heat service availability in tests.ci.osresources<commit_after># All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""List and compare most used OpenStack cloud resources."""

import argparse
import json
import subprocess
import sys

from rally.common.plugin import discover
from rally import consts
from rally import osclients

class ResourceManager(object):

    REQUIRED_SERVICE = None
    REPR_KEYS = ("id", "name", "tenant_id", "zone", "zoneName", "pool")

    def __init__(self, clients):
        self.clients = clients

    def is_available(self):
        if self.REQUIRED_SERVICE:
            return self.REQUIRED_SERVICE in self.clients.services().values()
        return True

    @property
    def client(self):
        return getattr(self.clients, self.__class__.__name__.lower())()

    def get_resources(self):
        all_resources = []
        cls = self.__class__.__name__.lower()
        for prop in dir(self):
            if not prop.startswith("list_"):
                continue
            f = getattr(self, prop)
            resources = f() or []
            resource_name = prop[5:][:-1]

            for res in resources:
                res_repr = []
                for key in self.REPR_KEYS + (resource_name,):
                    if isinstance(res, dict):
                        value = res.get(key)
                    else:
                        value = getattr(res, key, None)
                    if value:
                        res_repr.append("%s:%s" % (key, value))
                if not res_repr:
                    raise ValueError("Failed to represent resource %r" % res)

                all_resources.append(
                    "%s %s %s" % (cls, resource_name, " ".join(res_repr)))
        return all_resources

class Keystone(ResourceManager):

    def list_users(self):
        return self.client.users.list()

    def list_tenants(self):
        return self.client.tenants.list()

    def list_roles(self):
        return self.client.roles.list()

class Nova(ResourceManager):

    def list_flavors(self):
        return self.client.flavors.list()

    def list_floating_ip_pools(self):
        return self.client.floating_ip_pools.list()

    def list_floating_ips(self):
        return self.client.floating_ips.list()

    def list_images(self):
        return self.client.images.list()

    def list_keypairs(self):
        return self.client.keypairs.list()

    def list_networks(self):
        return self.client.networks.list()

    def list_security_groups(self):
        return self.client.security_groups.list(
            search_opts={"all_tenants": True})

    def list_servers(self):
        return self.client.servers.list(
            search_opts={"all_tenants": True})

    def list_services(self):
        return self.client.services.list()

    def list_availability_zones(self):
        return self.client.availability_zones.list()

class Neutron(ResourceManager):

    REQUIRED_SERVICE = consts.Service.NEUTRON

    def has_extension(self, name):
        extensions = self.client.list_extensions().get("extensions", [])
        return any(ext.get("alias") == name for ext in extensions)

    def list_networks(self):
        return self.client.list_networks()["networks"]

    def list_subnets(self):
        return self.client.list_subnets()["subnets"]

    def list_routers(self):
        return self.client.list_routers()["routers"]

    def list_ports(self):
        return self.client.list_ports()["ports"]

    def list_floatingips(self):
        return self.client.list_floatingips()["floatingips"]

    def list_security_groups(self):
        return self.client.list_security_groups()["security_groups"]

    def list_health_monitors(self):
        if self.has_extension("lbaas"):
            return self.client.list_health_monitors()["health_monitors"]

    def list_pools(self):
        if self.has_extension("lbaas"):
            return self.client.list_pools()["pools"]

    def list_vips(self):
        if self.has_extension("lbaas"):
            return self.client.list_vips()["vips"]

class Glance(ResourceManager):

    def list_images(self):
        return self.client.images.list()

class Heat(ResourceManager):

    REQUIRED_SERVICE = consts.Service.HEAT

    def list_resource_types(self):
        return self.client.resource_types.list()

    def list_stacks(self):
        return self.client.stacks.list()

class Cinder(ResourceManager):

    def list_availability_zones(self):
        return self.client.availability_zones.list()

    def list_backups(self):
        return self.client.backups.list()

    def list_volume_snapshots(self):
        return self.client.volume_snapshots.list()

    def list_volume_types(self):
        return self.client.volume_types.list()

    def list_volumes(self):
        return self.client.volumes.list(
            search_opts={"all_tenants": True})

class CloudResources(object):
    """List and compare cloud resources.

    resources = CloudResources(auth_url=..., ...)
    saved_list = resources.list()

    # Do something with the cloud ...

    changes = resources.compare(saved_list)
    has_changed = any(changes)
    removed, added = changes
    """

    def __init__(self, **kwargs):
        endpoint = osclients.objects.Endpoint(**kwargs)
        self.clients = osclients.Clients(endpoint)

    def _deduplicate(self, lst):
        """Change list duplicates to make all items unique.

        >>> resources._deduplicate(["a", "b", "c", "b", "b"])
        >>> ['a', 'b', 'c', 'b (duplicate 1)', 'b (duplicate 2)'
        """
        deduplicated_list = []
        for value in lst:
            if value in deduplicated_list:
                ctr = 0
                try_value = value
                while try_value in deduplicated_list:
                    ctr += 1
                    try_value = "%s (duplicate %i)" % (value, ctr)
                value = try_value
            deduplicated_list.append(value)
        return deduplicated_list

    def list(self):
        managers_classes = discover.itersubclasses(ResourceManager)
        resources = []
        for cls in managers_classes:
            manager = cls(self.clients)
            if manager.is_available():
                resources.extend(manager.get_resources())
        return sorted(self._deduplicate(resources))

    def compare(self, with_list):
        saved_resources = set(with_list)
        current_resources = set(self.list())
        removed = saved_resources - current_resources
        added = current_resources - saved_resources

        return sorted(list(removed)), sorted(list(added))

def main():

    parser = argparse.ArgumentParser(
        description=("Save list of OpenStack cloud resources or compare "
                     "with previously saved list."))
    parser.add_argument("--credentials",
                        type=argparse.FileType("r"),
                        metavar="<path/to/credentials.json>",
                        help="cloud credentials in JSON format")
    group = parser.add_mutually_exclusive_group(required=True)
    group.add_argument("--dump-list",
                       type=argparse.FileType("w"),
                       metavar="<path/to/output/list.json>",
                       help="dump resources to given file in JSON format")
    group.add_argument("--compare-with-list",
                       type=argparse.FileType("r"),
                       metavar="<path/to/existent/list.json>",
                       help=("compare current resources with a list from "
                             "given JSON file"))
    args = parser.parse_args()

    if args.credentials:
        config = json.load(args.credentials)
    else:
        config = json.loads(subprocess.check_output(["rally", "deployment",
                                                     "config"]))
        config.update(config.pop("admin"))
        del config["type"]

    resources = CloudResources(**config)

    if args.dump_list:
        resources_list = resources.list()
        json.dump(resources_list, args.dump_list, indent=2)
    elif args.compare_with_list:
        given_list = json.load(args.compare_with_list)
        changes = resources.compare(with_list=given_list)
        removed, added = changes
        sys.stdout.write(
            json.dumps({"removed": removed, "added": added}, indent=2))
        if any(changes):
            return 0  # `1' will fail gate job
    return 0

if __name__ == "__main__":
    sys.exit(main())
<|endoftext|>


<commit_before># Copyright 2020  Ternaris.
# SPDX-License-Identifier: AGPL-3.0-only

import warnings
from dataclasses import dataclass
from typing import Any

@dataclass
class Info:
    module: str
    version: str
    obj: Any
    msg: str = None

def dir_and_getattr(module, dunder_all, dct):
    assert all(x.module == module for x in dct.values())
    allattrs = tuple(set(dunder_all) | dct.keys())

    def __dir__():
        return allattrs

    def __getattr__(name):
        info = dct.get(name)
        if info is None:
            raise AttributeError(f'module {module} has no attribute {name}')

        msg = ' '.join([
            f'{module}.{name} will be removed in {info.version}.',
            info.msg or ' Please let us know if this is an issue for you.',
        ])
        warnings.warn(msg, FutureWarning, stacklevel=2)
        return info.obj

    return __dir__, __getattr__
<commit_msg>Don't show deprecated module attributes in __dir__<commit_after># Copyright 2020  Ternaris.
# SPDX-License-Identifier: AGPL-3.0-only

import warnings
from dataclasses import dataclass
from typing import Any

@dataclass
class Info:
    module: str
    version: str
    obj: Any
    msg: str = None

def make_getattr(module, dct):
    assert all(x.module == module for x in dct.values())

    def __getattr__(name):
        info = dct.get(name)
        if info is None:
            raise AttributeError(f'module {module} has no attribute {name}')

        msg = ' '.join([
            f'{module}.{name} will be removed in {info.version}.',
            info.msg or ' Please let us know if this is an issue for you.',
        ])
        warnings.warn(msg, FutureWarning, stacklevel=2)
        return info.obj

    return __getattr__
<|endoftext|>


<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from absl.testing import absltest
from absl.testing import parameterized

import numpy as onp

from jax import test_util as jtu
from jax.experimental.callback import (
    callback_transform, find_by_value, rewrite, FoundValue)
import jax.numpy as np
from jax import lax

from jax.config import config
config.parse_flags_with_absl()

class CallbackTest(jtu.JaxTestCase):
  @parameterized.named_parameters(jtu.cases_from_list(
      {'testcase_name': '_value={}'.format(value), 'value': value}
      for value in [np.inf, np.nan]))
  def testFindByValueFound(self, value):
    def f(x):
      y = x ** 2
      z = 1 - y
      r = 1 / z
      return r * 0

    with self.assertRaises(FoundValue):
      find_by_value(f, value)(np.array([1.0, 2.0, 3.0]))

  @parameterized.named_parameters(jtu.cases_from_list(
      {'testcase_name': '_value={}'.format(value), 'value': value}
      for value in [np.inf, np.nan]))
  def testFindByValueNotFound(self, value):
    def f(x):
      y = x ** 2
      z = 1 - y
      return z

    find_by_value(f, value)(np.array([1.0, 2.0, 3.0]))

  def testRewrite(self):
    def f(x):
      return x * 2

    x = np.array([2.0, 4.0])
    self.assertAllClose(f(x), np.array([4.0, 8.0]), True)

    self.assertAllClose(
        rewrite(f, {lax.mul_p: lambda x, y: x + y})(x),
        np.array([4.0, 6.0]), True)

if __name__ == "__main__":
  absltest.main()
<commit_msg>Added tests of callback_transform(jit(fn)).<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from absl.testing import absltest
from absl.testing import parameterized

import numpy as onp

from jax import test_util as jtu
from jax.experimental.callback import (
    callback_transform, find_by_value, rewrite, FoundValue)
import jax.numpy as np
from jax import lax
from jax import jit

from jax.config import config
config.parse_flags_with_absl()

class CallbackTest(jtu.JaxTestCase):
  @parameterized.named_parameters(jtu.cases_from_list(
      {'testcase_name': '_value={}'.format(value), 'value': value}
      for value in [np.inf, np.nan]))
  def testFindByValueFound(self, value):
    def f(x):
      y = x ** 2
      z = 1 - y
      r = 1 / z
      return r * 0

    with self.assertRaises(FoundValue):
      find_by_value(f, value)(np.array([1.0, 2.0, 3.0]))

  @parameterized.named_parameters(jtu.cases_from_list(
      {'testcase_name': '_value={}'.format(value), 'value': value}
      for value in [np.inf, np.nan]))
  def testFindByValueFoundJIT(self, value):
    def f(x):
      @jit
      def g(x):
        y = x ** 2
        z = 1 - y
        r = 1 / z
        return r * 0
      return g(x)
    with self.assertRaises(FoundValue):
      find_by_value(f, value)(np.array([1.0, 2.0, 3.0]))

  @parameterized.named_parameters(jtu.cases_from_list(
      {'testcase_name': '_value={}'.format(value), 'value': value}
      for value in [np.inf, np.nan]))
  def testFindByValueNotFound(self, value):
    def f(x):
      y = x ** 2
      z = 1 - y
      return z

    find_by_value(f, value)(np.array([1.0, 2.0, 3.0]))

  def testRewrite(self):
    def f(x):
      return x * 2

    x = np.array([2.0, 4.0])
    self.assertAllClose(f(x), np.array([4.0, 8.0]), True)

    self.assertAllClose(
        rewrite(f, {lax.mul_p: lambda x, y: x + y})(x),
        np.array([4.0, 6.0]), True)

  def testRewriteJIT(self):
    def f(x):
      @jit
      def g(x):
        return x * 2
      return g(x)

    x = np.array([2.0, 4.0])
    self.assertAllClose(f(x), np.array([4.0, 8.0]), True)

    self.assertAllClose(
        rewrite(f, {lax.mul_p: lambda x, y: x + y})(x),
        np.array([4.0, 6.0]), True)

if __name__ == "__main__":
  absltest.main()
<|endoftext|>