<commit_before>import requests
import igraph

if __name__ == '__main__':

    # Some constants that determine what data to fetch
    start_date = 1383264000 # November 1, 2013
    end_date = 1383350400   # November 2, 2013
    site = 'stackoverflow'
    tag_list = ['python']

    graph = BuildGraph(graph, start_date, end_date, site, tags)

def BuildGraph(start_date, end_date,
               site='stackoverflow', tags = [])
    """ Builds a graph from data from StackExchange with the specified
        site and tags, between the start and end date. The graphs's
        vertices are users and edges are if a user has answered another
        user's question. It is therefore directed. """

    # Create a new graph
    graph = igraph.Graph(directed = True)

def GetQuestionRequestString(page, fromdate, todate, tags, site):
    """ Returns the StackExchange API request string for questions
        with the specified arguments """
    req = "/2.1/questions?page=%i&fromdate=%i&todate=%i"%(page,
                                                          fromdate,
                                                          todate)
    req += "order=desc&sort=activity&tagged=%s&site=%s"%(';'.join(tags),
                                                         site)
    return req
<commit_msg>Filling ou the functions.<commit_after>import requests
import igraph

if __name__ == '__main__':

    # Some constants that determine what data to fetch
    start_date = 1383264000 # November 1, 2013
    end_date = 1383350400   # November 2, 2013
    site = 'stackoverflow'
    tag_list = ['python']

    graph = BuildGraph(graph, start_date, end_date, site, tags)

def BuildGraph(start_date, end_date,
               site='stackoverflow', tags = [],
               base_url='http://api.stackexhange.com',
               start_page = 1,
               graph = None)
    """ Builds a graph from data from StackExchange with the specified
        site and tags, between the start and end date. The graphs's
        vertices are users and edges are if a user has answered another
        user's question. It is therefore directed.
        Can also add to an existing graph. This and the start_page argument
        are useful if quota limits prevented grabbing all the data """

    if not graph:
        # Create a new graph
        graph = igraph.Graph(directed = True)

    # Keep track of how we've grab from StackExchange
    # and if we've grabbed everything
    quota_remaining = 10000
    has_more = True
    page = start_page

    while quota_remaining > 0 and has_more:
        # Just keep grabbing questions and looking at their answers
        req = GetQuestionRequestString(page, start_date, end_date
                                       tags, site)
        url = base_url + req
        r = requests.get(url)
        data = r.json()
        questions = data.get('items', []) # empty list if no questions returned
        for question in questions:
            AddQuestionToGraph(question, graph)

        # update for next iteration
        if quota_remaining > 0:
            page += 1
        has_more = data['has_more']
        quota_remaining = data['quota_remaining']

    print "Processed %i pages."%(page)
    print "Graph summary:"
    igraph.summary(graph)

def GetQuestionRequestString(page, fromdate, todate, tags, site):
    """ Returns the StackExchange API request string for questions
        with the specified arguments """
    req = "/2.1/questions?page=%i&fromdate=%i&todate=%i"%(page,
                                                          fromdate,
                                                          todate)
    req += "order=desc&sort=activity&tagged=%s&site=%s"%(';'.join(tags),
                                                         site)
    return req
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import os

try:
    from collections import OrderedDict
except ImportError:
    from django.utils.datastructures import SortedDict as OrderedDict

from django.contrib.staticfiles import finders
from django.contrib.staticfiles.storage import staticfiles_storage

from pipeline.finders import PipelineFinder

class Collector(object):
    def __init__(self, storage=None):
        if storage is None:
            storage = staticfiles_storage
        self.storage = storage

    def clear(self, path=""):
        dirs, files = self.storage.listdir(path)
        for f in files:
            fpath = os.path.join(path, f)
            self.storage.delete(fpath)
        for d in dirs:
            self.clear(os.path.join(path, d))

    def collect(self):
        found_files = OrderedDict()
        for finder in finders.get_finders():
            # Ignore our finder to avoid looping
            if isinstance(finder, PipelineFinder):
                continue
            for path, storage in finder.list(['CVS', '.*', '*~']):
                # Prefix the relative path if the source storage contains it
                if getattr(storage, 'prefix', None):
                    prefixed_path = os.path.join(storage.prefix, path)
                else:
                    prefixed_path = path
                if prefixed_path not in found_files:
                    found_files[prefixed_path] = (storage, path)
                    self.copy_file(path, prefixed_path, storage)

    def copy_file(self, path, prefixed_path, source_storage):
        # Delete the target file if needed or break
        if not self.delete_file(path, prefixed_path, source_storage):
            return
        # Finally start copying
        with source_storage.open(path) as source_file:
            self.storage.save(prefixed_path, source_file)

    def delete_file(self, path, prefixed_path, source_storage):
        if self.storage.exists(prefixed_path):
            try:
                # When was the target file modified last time?
                target_last_modified = self.storage.modified_time(prefixed_path)
            except (OSError, NotImplementedError, AttributeError):
                # The storage doesn't support ``modified_time`` or failed
                pass
            else:
                try:
                    # When was the source file modified last time?
                    source_last_modified = source_storage.modified_time(path)
                except (OSError, NotImplementedError, AttributeError):
                    pass
                else:
                    # Skip the file if the source file is younger
                    # Avoid sub-second precision
                    if (target_last_modified.replace(microsecond=0)
                            >= source_last_modified.replace(microsecond=0)):
                            return False
            # Then delete the existing file if really needed
            self.storage.delete(prefixed_path)
        return True

default_collector = Collector()
<commit_msg>Remove obvious 2.6 import.<commit_after>from __future__ import unicode_literals

import os

from collections import OrderedDict

from django.contrib.staticfiles import finders
from django.contrib.staticfiles.storage import staticfiles_storage

from pipeline.finders import PipelineFinder

class Collector(object):
    def __init__(self, storage=None):
        if storage is None:
            storage = staticfiles_storage
        self.storage = storage

    def clear(self, path=""):
        dirs, files = self.storage.listdir(path)
        for f in files:
            fpath = os.path.join(path, f)
            self.storage.delete(fpath)
        for d in dirs:
            self.clear(os.path.join(path, d))

    def collect(self):
        found_files = OrderedDict()
        for finder in finders.get_finders():
            # Ignore our finder to avoid looping
            if isinstance(finder, PipelineFinder):
                continue
            for path, storage in finder.list(['CVS', '.*', '*~']):
                # Prefix the relative path if the source storage contains it
                if getattr(storage, 'prefix', None):
                    prefixed_path = os.path.join(storage.prefix, path)
                else:
                    prefixed_path = path
                if prefixed_path not in found_files:
                    found_files[prefixed_path] = (storage, path)
                    self.copy_file(path, prefixed_path, storage)

    def copy_file(self, path, prefixed_path, source_storage):
        # Delete the target file if needed or break
        if not self.delete_file(path, prefixed_path, source_storage):
            return
        # Finally start copying
        with source_storage.open(path) as source_file:
            self.storage.save(prefixed_path, source_file)

    def delete_file(self, path, prefixed_path, source_storage):
        if self.storage.exists(prefixed_path):
            try:
                # When was the target file modified last time?
                target_last_modified = self.storage.modified_time(prefixed_path)
            except (OSError, NotImplementedError, AttributeError):
                # The storage doesn't support ``modified_time`` or failed
                pass
            else:
                try:
                    # When was the source file modified last time?
                    source_last_modified = source_storage.modified_time(path)
                except (OSError, NotImplementedError, AttributeError):
                    pass
                else:
                    # Skip the file if the source file is younger
                    # Avoid sub-second precision
                    if (target_last_modified.replace(microsecond=0)
                            >= source_last_modified.replace(microsecond=0)):
                            return False
            # Then delete the existing file if really needed
            self.storage.delete(prefixed_path)
        return True

default_collector = Collector()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import codecs, sys
from memrise import Course, COURSE_URL
from gtts import gTTS

def dump_tts(*, course_url : str):
    """
    :course_url:   course URL
    """

    course = Course(course_url=course_url)
    os.makedirs(course.name.replace(' ', "\\ "), exist_ok=True)
    print(f"* {course.name}\n\n")

    for level_url, title in course.levels:
        print("*** %s (%s)" % (title, level_url), flush=True)
        if sys.platform.startswith("win"):
            # replace illegal symbols in the windows file name
            title = title.replace(':', '-')

        output_dir = os.path.join(course.name, title)
        if not no_audio:
            os.makedirs(output_dir, exist_ok=True)

        with codecs.open(os.path.join(course.name, title) + ".csv", "w", encoding="utf-8") as level_file:
            for card in course.cards(level_url=level_url):
                card = list(card)
                word = card[0]
                level_file.write('\t'.join(card))
                level_file.write('\n')
                file_name = os.path.join(output_dir, word) + ".mp3"
                if not no_audio and not os.path.exists(file_name):
                    tts = gTTS(word, lang="ko")
                    tts.save(file_name)

if __name__ == "__main__":
    sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
    no_audio = ("--no-audio" in sys.argv)
    if no_audio:
        sys.argv.pop(sys.argv.index("--no-audio"))
    course_url = COURSE_URL if len(sys.argv) < 2 else sys.argv[1]
    dump_tts(course_url=course_url)
<commit_msg>deal with '/'..<commit_after>#!/usr/bin/env python
import os
import codecs, sys
from memrise import Course, COURSE_URL
from gtts import gTTS

def dump_tts(*, course_url : str):
    """
    :course_url:   course URL
    """

    course = Course(course_url=course_url)
    course_dir_name = course.name #.replace(' ', "\\ ")
    os.makedirs(course_dir_name, exist_ok=True)
    print(f"* {course.name}\n\n")

    for level_url, title in course.levels:
        print("*** %s (%s)" % (title, level_url), flush=True)
        if sys.platform.startswith("win"):
            # replace illegal symbols in the windows file name
            title = title.replace(':', '-')

        output_dir = os.path.join(course_dir_name, title)
        if not no_audio:
            os.makedirs(output_dir, exist_ok=True)

        with codecs.open(os.path.join(course.name, title) + ".csv", "w", encoding="utf-8") as level_file:
            for card in course.cards(level_url=level_url):
                card = list(card)
                word = card[0]
                level_file.write('\t'.join(card))
                level_file.write('\n')
                file_name = os.path.join(output_dir, word.replace('/', "|")) + ".mp3"
                if not no_audio and not os.path.exists(file_name):
                    tts = gTTS(word, lang="ko")
                    tts.save(file_name)

if __name__ == "__main__":
    sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
    no_audio = ("--no-audio" in sys.argv)
    if no_audio:
        sys.argv.pop(sys.argv.index("--no-audio"))
    course_url = COURSE_URL if len(sys.argv) < 2 else sys.argv[1]
    dump_tts(course_url=course_url)
<|endoftext|>


<commit_before>import argparse
import copy
import random
from collections import defaultdict

import numpy as np
import torch
from scipy.sparse import issparse
from torch.utils.data import Dataset

class MetalDataset(Dataset):
    """A dataset that group each item in X with its label from Y

    Args:
        X: an n-dim iterable of items
        Y: a torch.Tensor of labels
            This may be predicted (int) labels [n] or probabilistic (float) labels [n, k]
    """

    def __init__(self, X, Y):
        self.X = X
        self.Y = Y
        assert len(X) == len(Y)

    def __getitem__(self, index):
        return tuple([self.X[index], self.Y[index]])

    def __len__(self):
        return len(self.X)

def rargmax(x, eps=1e-8):
    """Argmax with random tie-breaking

    Args:
        x: a 1-dim numpy array
    Returns:
        the argmax index
    """
    idxs = np.where(abs(x - np.max(x, axis=0)) < eps)[0]
    return np.random.choice(idxs)

def pred_to_prob(Y_h, k):
    """Converts a 1D tensor of predicted labels into a 2D tensor of probabilistic labels

    Args:
        Y_h: an [n], or [n,1] tensor of predicted (int) labels in {1,...,k}
        k: the largest possible label in Y_h
    Returns:
        Y_s: a torch.FloatTensor of shape [n, k] where Y_s[i, j-1] is the probabilistic
            label for item i and label j
    """
    Y_h = Y_h.clone()
    Y_h = Y_h.squeeze()
    assert Y_h.dim() == 1
    assert (Y_h >= 1).all()
    assert (Y_h <= k).all()
    n = Y_h.shape[0]
    Y_s = torch.zeros((n, k), dtype=Y_h.dtype, device=Y_h.device)
    for i, j in enumerate(Y_h):
        Y_s[i, j - 1] = 1.0
    return Y_s

def arraylike_to_numpy(array_like):
    """Convert a 1d array-like (e.g,. list, tensor, etc.) to an np.ndarray"""

    orig_type = type(array_like)

    # Convert to np.ndarray
    if isinstance(array_like, np.ndarray):
        pass
    elif isinstance(array_like, list):
        array_like = np.array(array_like)
    elif issparse(array_like):
        array_like = array_like.toarray()
    elif isinstance(array_like, torch.Tensor):
        array_like = array_like.numpy()
    elif not isinstance(array_like, np.ndarray):
        array_like = np.array(array_like)
    else:
        msg = f"Input of type {orig_type} could not be converted to 1d " "np.ndarray"
        raise ValueError(msg)

    # Correct shape
    if (array_like.ndim > 1) and (1 in array_like.shape):
        array_like = array_like.flatten()
    if array_like.ndim != 1:
        raise ValueError("Input could not be converted to 1d np.array")

    # Convert to ints
    if any(array_like % 1):
        raise ValueError("Input contains at least one non-integer value.")
    array_like = array_like.astype(np.dtype(int))

    return array_like

def convert_labels(Y, source, dest):
    """Convert a matrix from one label type to another

    Args:
        Y: A np.ndarray or torch.Tensor of labels (ints)
        source: The convention the labels are currently expressed in
        dest: The convention to convert the labels to

    Conventions:
        'categorical': [0: abstain, 1: positive, 2: negative]
        'plusminus': [0: abstain, 1: positive, -1: negative]
        'onezero': [0: negative, 1: positive]

    Note that converting to 'onezero' will combine abstain and negative labels.
    """
    if Y is None:
        return Y
    if isinstance(Y, np.ndarray):
        Y = Y.copy()
        assert isinstance(Y, int)
    elif isinstance(Y, torch.Tensor):
        Y = Y.clone()
        assert np.sum(Y.numpy() - Y.numpy().astype(int)) == 0.0
    else:
        raise ValueError("Unrecognized label data type.")
    negative_map = {"categorical": 2, "plusminus": -1, "onezero": 0}
    Y[Y == negative_map[source]] = negative_map[dest]
    return Y

def plusminus_to_categorical(Y):
    return convert_labels(Y, "plusminus", "categorical")

def categorical_to_plusminus(Y):
    return convert_labels(Y, "categorical", "plusminus")

def label_matrix_to_one_hot(L, k=None):
    """Converts a 2D [n,m] label matrix into an [n,m,k] one hot 3D tensor

    Note that in the returned 3D matrix, abstain votes continue to be
    represented by 0s, not 1s.

    Args:
        L: a [n,m] label matrix with categorical labels (0 = abstain)
        k: the number of classes that could appear in L
            if None, k is inferred as the max element in L
    """
    n, m = L.shape
    if k is None:
        k = L.max()
    L_onehot = torch.zeros(n, m, k + 1)
    for i, row in enumerate(L):
        for j, k in enumerate(row):
            if k > 0:
                L_onehot[i, j, k - 1] = 1
    return L_onehot

def recursive_merge_dicts(x, y, misses="report", verbose=None):
    """
    Merge dictionary y into a copy of x, overwriting elements of x when there
    is a conflict, except if the element is a dictionary, in which case recurse.

    misses: what to do if a key in y is not in x
        'insert'    -> set x[key] = value
        'exception' -> raise an exception
        'report'    -> report the name of the missing key
        'ignore'    -> do nothing

    TODO: give example here (pull from tests)
    """

    def recurse(x, y, misses="report", verbose=1):
        found = True
        for k, v in y.items():
            found = False
            if k in x:
                found = True
                if isinstance(x[k], dict):
                    if not isinstance(v, dict):
                        msg = f"Attempted to overwrite dict {k} with " f"non-dict: {v}"
                        raise ValueError(msg)
                    recurse(x[k], v, misses, verbose)
                else:
                    if x[k] == v:
                        msg = f"Reaffirming {k}={x[k]}"
                    else:
                        msg = f"Overwriting {k}={x[k]} to {k}={v}"
                        x[k] = v
                    if verbose > 1 and k != "verbose":
                        print(msg)
            else:
                for kx, vx in x.items():
                    if isinstance(vx, dict):
                        found = recurse(vx, {k: v}, misses="ignore", verbose=verbose)
                    if found:
                        break
            if not found:
                msg = f'Could not find kwarg "{k}" in destination dict.'
                if misses == "insert":
                    x[k] = v
                    if verbose > 1:
                        print(f"Added {k}={v} from second dict to first")
                elif misses == "exception":
                    raise ValueError(msg)
                elif misses == "report":
                    print(msg)
                else:
                    pass
        return found

    # If verbose is not provided, look for an value in y first, then x
    # (Do this because 'verbose' kwarg is often inside one or both of x and y)
    if verbose is None:
        verbose = y.get("verbose", x.get("verbose", 1))

    z = copy.deepcopy(x)
    recurse(z, y, misses, verbose)
    return z

def recursive_transform(x, test_func, transform):
    """Applies a transformation recursively to each member of a dictionary

    Args:
        x: a (possibly nested) dictionary
        test_func: a function that returns whether this element should be transformed
        transform: a function that transforms a value
    """
    for k, v in x.items():
        if test_func(v):
            x[k] = transform(v)
        if isinstance(v, dict):
            recursive_transform(v, test_func, transform)
    return x

def add_flags_from_config(parser, config_dict):
    """
    Adds a flag (and default value) to an ArgumentParser for each parameter in a config
    """

    def OrNone(default):
        def func(x):
            return None if x.lower() == "none" else type(default)(x)

        return func

    for param in config_dict:
        # Blacklist certain config parameters from being added as flags
        if param in ["verbose"]:
            continue

        default = config_dict[param]
        try:
            if isinstance(default, dict):
                parser = add_flags_from_config(parser, default)
            elif isinstance(default, list):
                if len(default) > 0:
                    # pass a list as argument
                    parser.add_argument(
                        f"--{param}",
                        action="append",
                        type=type(default[0]),
                        default=default,
                    )
                else:
                    parser.add_argument(f"--{param}", action="append", default=default)
            else:
                parser.add_argument(f"--{param}", type=OrNone(default), default=default)
        except argparse.ArgumentError:
            print(
                f"Could not add flag for param {param} because it was already present."
            )
    return parser

def split_data(
    *inputs,
    splits=[0.5, 0.5],
    shuffle=True,
    stratify_by=None,
    index_only=False,
    seed=None,
):
    """Splits inputs into multiple splits of defined sizes

    Args:
        inputs: correlated tuples/lists/arrays/matrices/tensors to split
        splits: list containing split sizes (fractions or counts);
        shuffle: if True, shuffle the data before splitting
        stratify_by: (None or an input) if not None, use these labels to
            stratify the splits (separating the data into groups by these
            labels and sampling from those, rather than from the population at
            large); overrides shuffle
        index_only: if True, return only the indices of the new splits, not the
            split data itself
        seed: (int) random seed

    Example usage:
        Ls, Xs, Ys = split_data(L, X, Y, splits=[0.8, 0.1, 0.1])
        OR
        assignments = split_data(Y, splits=[0.8, 0.1, 0.1], index_only=True)

    Note: This is very similar to scikit-learn's train_test_split() method,
        but with support for more than two splits.
    """

    def fractions_to_counts(fracs, n):
        """Converts a list of fractions to a list of counts that sum to n"""
        counts = [int(np.round(n * frac)) for frac in fracs]
        # Ensure sum of split counts sums to n
        counts[-1] = n - sum(counts[:-1])
        return counts

    def slice_data(data, indices):
        if isinstance(data, list) or isinstance(data, tuple):
            return [d for i, d in enumerate(data) if i in set(indices)]
        else:
            try:
                # Works for np.ndarray, scipy.sparse, torch.Tensor
                return data[indices]
            except TypeError:
                raise Exception(
                    f"split_data() currently only accepts inputs "
                    f"of type tuple, list, np.ndarray, scipy.sparse, or "
                    f"torch.Tensor; not {type(data)}"
                )

    # Setting random seed
    if seed is not None:
        random.seed(seed)

    try:
        n = len(inputs[0])
    except TypeError:
        n = inputs[0].shape[0]
    num_splits = len(splits)

    # Check splits for validity and convert to fractions
    if all(isinstance(x, int) for x in splits):
        if not sum(splits) == n:
            raise ValueError(
                f"Provided split counts must sum to n ({n}), not {sum(splits)}."
            )
        fracs = [count / n for count in splits]

    elif all(isinstance(x, float) for x in splits):
        if not sum(splits) == 1.0:
            raise ValueError(f"Split fractions must sum to 1.0, not {sum(splits)}.")
        fracs = splits

    else:
        raise ValueError("Splits must contain all ints or all floats.")

    # Make sampling pools
    if stratify_by is None:
        pools = [np.arange(n)]
    else:
        pools = defaultdict(list)
        for i, val in enumerate(stratify_by):
            pools[val].append(i)
        pools = list(pools.values())

    # Make index assignments
    assignments = [[] for _ in range(num_splits)]
    for pool in pools:
        if shuffle or stratify_by is not None:
            random.shuffle(pool)

        counts = fractions_to_counts(fracs, len(pool))
        counts.insert(0, 0)
        cum_counts = np.cumsum(counts)
        for i in range(num_splits):
            assignments[i].extend(pool[cum_counts[i] : cum_counts[i + 1]])

    if index_only:
        return assignments
    else:
        outputs = []
        for data in inputs:
            data_splits = []
            for split in range(num_splits):
                data_splits.append(slice_data(data, assignments[split]))
            outputs.append(data_splits)

        if len(outputs) == 1:
            return outputs[0]
        else:
            return outputs

# DEPRECATION: This is replaced by move_to_device
def place_on_gpu(data):
    """Utility to place data on GPU, where data could be a torch.Tensor, a tuple
    or list of Tensors, or a tuple or list of tuple or lists of Tensors"""
    data_type = type(data)
    if data_type in (list, tuple):
        data = [place_on_gpu(data[i]) for i in range(len(data))]
        data = data_type(data)
        return data
    elif isinstance(data, torch.Tensor):
        return data.cuda()
    else:
        return ValueError(f"Data type {type(data)} not recognized.")

def move_to_device(obj, device=-1):
    """
    Given a structure (possibly) containing Tensors on the CPU,
    move all the Tensors to the specified GPU (or do nothing, if they should be on the CPU).

    device = -1 -> "cpu"
    device =  0 -> "cuda:0"

    """
    if device < 0 or not torch.cuda.is_available():
        return obj
    elif isinstance(obj, torch.Tensor):
        return obj.cuda(device)
    elif isinstance(obj, dict):
        return {key: move_to_device(value, device) for key, value in obj.items()}
    elif isinstance(obj, list):
        return [move_to_device(item, device) for item in obj]
    elif isinstance(obj, tuple):
        return tuple([move_to_device(item, device) for item in obj])
    else:
        return obj

def set_seed(seed):
    seed = int(seed)
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.backends.cudnn.enabled = True  # Is this necessary?
        torch.cuda.manual_seed(seed)
<commit_msg>Fix automatic flag conversion for cases where default is None but value is non-None<commit_after>import argparse
import copy
import random
from collections import defaultdict

import numpy as np
import torch
from scipy.sparse import issparse
from torch.utils.data import Dataset

class MetalDataset(Dataset):
    """A dataset that group each item in X with its label from Y

    Args:
        X: an n-dim iterable of items
        Y: a torch.Tensor of labels
            This may be predicted (int) labels [n] or probabilistic (float) labels [n, k]
    """

    def __init__(self, X, Y):
        self.X = X
        self.Y = Y
        assert len(X) == len(Y)

    def __getitem__(self, index):
        return tuple([self.X[index], self.Y[index]])

    def __len__(self):
        return len(self.X)

def rargmax(x, eps=1e-8):
    """Argmax with random tie-breaking

    Args:
        x: a 1-dim numpy array
    Returns:
        the argmax index
    """
    idxs = np.where(abs(x - np.max(x, axis=0)) < eps)[0]
    return np.random.choice(idxs)

def pred_to_prob(Y_h, k):
    """Converts a 1D tensor of predicted labels into a 2D tensor of probabilistic labels

    Args:
        Y_h: an [n], or [n,1] tensor of predicted (int) labels in {1,...,k}
        k: the largest possible label in Y_h
    Returns:
        Y_s: a torch.FloatTensor of shape [n, k] where Y_s[i, j-1] is the probabilistic
            label for item i and label j
    """
    Y_h = Y_h.clone()
    Y_h = Y_h.squeeze()
    assert Y_h.dim() == 1
    assert (Y_h >= 1).all()
    assert (Y_h <= k).all()
    n = Y_h.shape[0]
    Y_s = torch.zeros((n, k), dtype=Y_h.dtype, device=Y_h.device)
    for i, j in enumerate(Y_h):
        Y_s[i, j - 1] = 1.0
    return Y_s

def arraylike_to_numpy(array_like):
    """Convert a 1d array-like (e.g,. list, tensor, etc.) to an np.ndarray"""

    orig_type = type(array_like)

    # Convert to np.ndarray
    if isinstance(array_like, np.ndarray):
        pass
    elif isinstance(array_like, list):
        array_like = np.array(array_like)
    elif issparse(array_like):
        array_like = array_like.toarray()
    elif isinstance(array_like, torch.Tensor):
        array_like = array_like.numpy()
    elif not isinstance(array_like, np.ndarray):
        array_like = np.array(array_like)
    else:
        msg = f"Input of type {orig_type} could not be converted to 1d " "np.ndarray"
        raise ValueError(msg)

    # Correct shape
    if (array_like.ndim > 1) and (1 in array_like.shape):
        array_like = array_like.flatten()
    if array_like.ndim != 1:
        raise ValueError("Input could not be converted to 1d np.array")

    # Convert to ints
    if any(array_like % 1):
        raise ValueError("Input contains at least one non-integer value.")
    array_like = array_like.astype(np.dtype(int))

    return array_like

def convert_labels(Y, source, dest):
    """Convert a matrix from one label type to another

    Args:
        Y: A np.ndarray or torch.Tensor of labels (ints)
        source: The convention the labels are currently expressed in
        dest: The convention to convert the labels to

    Conventions:
        'categorical': [0: abstain, 1: positive, 2: negative]
        'plusminus': [0: abstain, 1: positive, -1: negative]
        'onezero': [0: negative, 1: positive]

    Note that converting to 'onezero' will combine abstain and negative labels.
    """
    if Y is None:
        return Y
    if isinstance(Y, np.ndarray):
        Y = Y.copy()
        assert Y.dtype == np.int64
    elif isinstance(Y, torch.Tensor):
        Y = Y.clone()
        assert np.sum(Y.numpy() - Y.numpy().astype(int)) == 0.0
    else:
        raise ValueError("Unrecognized label data type.")
    negative_map = {"categorical": 2, "plusminus": -1, "onezero": 0}
    Y[Y == negative_map[source]] = negative_map[dest]
    return Y

def plusminus_to_categorical(Y):
    return convert_labels(Y, "plusminus", "categorical")

def categorical_to_plusminus(Y):
    return convert_labels(Y, "categorical", "plusminus")

def label_matrix_to_one_hot(L, k=None):
    """Converts a 2D [n,m] label matrix into an [n,m,k] one hot 3D tensor

    Note that in the returned 3D matrix, abstain votes continue to be
    represented by 0s, not 1s.

    Args:
        L: a [n,m] label matrix with categorical labels (0 = abstain)
        k: the number of classes that could appear in L
            if None, k is inferred as the max element in L
    """
    n, m = L.shape
    if k is None:
        k = L.max()
    L_onehot = torch.zeros(n, m, k + 1)
    for i, row in enumerate(L):
        for j, k in enumerate(row):
            if k > 0:
                L_onehot[i, j, k - 1] = 1
    return L_onehot

def recursive_merge_dicts(x, y, misses="report", verbose=None):
    """
    Merge dictionary y into a copy of x, overwriting elements of x when there
    is a conflict, except if the element is a dictionary, in which case recurse.

    misses: what to do if a key in y is not in x
        'insert'    -> set x[key] = value
        'exception' -> raise an exception
        'report'    -> report the name of the missing key
        'ignore'    -> do nothing

    TODO: give example here (pull from tests)
    """

    def recurse(x, y, misses="report", verbose=1):
        found = True
        for k, v in y.items():
            found = False
            if k in x:
                found = True
                if isinstance(x[k], dict):
                    if not isinstance(v, dict):
                        msg = f"Attempted to overwrite dict {k} with " f"non-dict: {v}"
                        raise ValueError(msg)
                    recurse(x[k], v, misses, verbose)
                else:
                    if x[k] == v:
                        msg = f"Reaffirming {k}={x[k]}"
                    else:
                        msg = f"Overwriting {k}={x[k]} to {k}={v}"
                        x[k] = v
                    if verbose > 1 and k != "verbose":
                        print(msg)
            else:
                for kx, vx in x.items():
                    if isinstance(vx, dict):
                        found = recurse(vx, {k: v}, misses="ignore", verbose=verbose)
                    if found:
                        break
            if not found:
                msg = f'Could not find kwarg "{k}" in destination dict.'
                if misses == "insert":
                    x[k] = v
                    if verbose > 1:
                        print(f"Added {k}={v} from second dict to first")
                elif misses == "exception":
                    raise ValueError(msg)
                elif misses == "report":
                    print(msg)
                else:
                    pass
        return found

    # If verbose is not provided, look for an value in y first, then x
    # (Do this because 'verbose' kwarg is often inside one or both of x and y)
    if verbose is None:
        verbose = y.get("verbose", x.get("verbose", 1))

    z = copy.deepcopy(x)
    recurse(z, y, misses, verbose)
    return z

def recursive_transform(x, test_func, transform):
    """Applies a transformation recursively to each member of a dictionary

    Args:
        x: a (possibly nested) dictionary
        test_func: a function that returns whether this element should be transformed
        transform: a function that transforms a value
    """
    for k, v in x.items():
        if test_func(v):
            x[k] = transform(v)
        if isinstance(v, dict):
            recursive_transform(v, test_func, transform)
    return x

def add_flags_from_config(parser, config_dict):
    """
    Adds a flag (and default value) to an ArgumentParser for each parameter in a config
    """

    def OrNone(default):
        def func(x):
            # Convert "none" to proper None object
            if x.lower() == "none":
                return None
            # If default is None (and x is not None), return x without conversion
            elif default is None:
                return x
            # Otherwise, default has non-None type; convert x to that type
            else:
                return type(default)(x)

        return func

    for param in config_dict:
        # Blacklist certain config parameters from being added as flags
        if param in ["verbose"]:
            continue

        default = config_dict[param]
        try:
            if isinstance(default, dict):
                parser = add_flags_from_config(parser, default)
            elif isinstance(default, list):
                if len(default) > 0:
                    # pass a list as argument
                    parser.add_argument(
                        f"--{param}",
                        action="append",
                        type=type(default[0]),
                        default=default,
                    )
                else:
                    parser.add_argument(f"--{param}", action="append", default=default)
            else:
                parser.add_argument(f"--{param}", type=OrNone(default), default=default)
        except argparse.ArgumentError:
            print(
                f"Could not add flag for param {param} because it was already present."
            )
    return parser

def split_data(
    *inputs,
    splits=[0.5, 0.5],
    shuffle=True,
    stratify_by=None,
    index_only=False,
    seed=None,
):
    """Splits inputs into multiple splits of defined sizes

    Args:
        inputs: correlated tuples/lists/arrays/matrices/tensors to split
        splits: list containing split sizes (fractions or counts);
        shuffle: if True, shuffle the data before splitting
        stratify_by: (None or an input) if not None, use these labels to
            stratify the splits (separating the data into groups by these
            labels and sampling from those, rather than from the population at
            large); overrides shuffle
        index_only: if True, return only the indices of the new splits, not the
            split data itself
        seed: (int) random seed

    Example usage:
        Ls, Xs, Ys = split_data(L, X, Y, splits=[0.8, 0.1, 0.1])
        OR
        assignments = split_data(Y, splits=[0.8, 0.1, 0.1], index_only=True)

    Note: This is very similar to scikit-learn's train_test_split() method,
        but with support for more than two splits.
    """

    def fractions_to_counts(fracs, n):
        """Converts a list of fractions to a list of counts that sum to n"""
        counts = [int(np.round(n * frac)) for frac in fracs]
        # Ensure sum of split counts sums to n
        counts[-1] = n - sum(counts[:-1])
        return counts

    def slice_data(data, indices):
        if isinstance(data, list) or isinstance(data, tuple):
            return [d for i, d in enumerate(data) if i in set(indices)]
        else:
            try:
                # Works for np.ndarray, scipy.sparse, torch.Tensor
                return data[indices]
            except TypeError:
                raise Exception(
                    f"split_data() currently only accepts inputs "
                    f"of type tuple, list, np.ndarray, scipy.sparse, or "
                    f"torch.Tensor; not {type(data)}"
                )

    # Setting random seed
    if seed is not None:
        random.seed(seed)

    try:
        n = len(inputs[0])
    except TypeError:
        n = inputs[0].shape[0]
    num_splits = len(splits)

    # Check splits for validity and convert to fractions
    if all(isinstance(x, int) for x in splits):
        if not sum(splits) == n:
            raise ValueError(
                f"Provided split counts must sum to n ({n}), not {sum(splits)}."
            )
        fracs = [count / n for count in splits]

    elif all(isinstance(x, float) for x in splits):
        if not sum(splits) == 1.0:
            raise ValueError(f"Split fractions must sum to 1.0, not {sum(splits)}.")
        fracs = splits

    else:
        raise ValueError("Splits must contain all ints or all floats.")

    # Make sampling pools
    if stratify_by is None:
        pools = [np.arange(n)]
    else:
        pools = defaultdict(list)
        for i, val in enumerate(stratify_by):
            pools[val].append(i)
        pools = list(pools.values())

    # Make index assignments
    assignments = [[] for _ in range(num_splits)]
    for pool in pools:
        if shuffle or stratify_by is not None:
            random.shuffle(pool)

        counts = fractions_to_counts(fracs, len(pool))
        counts.insert(0, 0)
        cum_counts = np.cumsum(counts)
        for i in range(num_splits):
            assignments[i].extend(pool[cum_counts[i] : cum_counts[i + 1]])

    if index_only:
        return assignments
    else:
        outputs = []
        for data in inputs:
            data_splits = []
            for split in range(num_splits):
                data_splits.append(slice_data(data, assignments[split]))
            outputs.append(data_splits)

        if len(outputs) == 1:
            return outputs[0]
        else:
            return outputs

# DEPRECATION: This is replaced by move_to_device
def place_on_gpu(data):
    """Utility to place data on GPU, where data could be a torch.Tensor, a tuple
    or list of Tensors, or a tuple or list of tuple or lists of Tensors"""
    data_type = type(data)
    if data_type in (list, tuple):
        data = [place_on_gpu(data[i]) for i in range(len(data))]
        data = data_type(data)
        return data
    elif isinstance(data, torch.Tensor):
        return data.cuda()
    else:
        return ValueError(f"Data type {type(data)} not recognized.")

def move_to_device(obj, device=-1):
    """
    Given a structure (possibly) containing Tensors on the CPU,
    move all the Tensors to the specified GPU (or do nothing, if they should be on the CPU).

    device = -1 -> "cpu"
    device =  0 -> "cuda:0"

    """
    if device < 0 or not torch.cuda.is_available():
        return obj
    elif isinstance(obj, torch.Tensor):
        return obj.cuda(device)
    elif isinstance(obj, dict):
        return {key: move_to_device(value, device) for key, value in obj.items()}
    elif isinstance(obj, list):
        return [move_to_device(item, device) for item in obj]
    elif isinstance(obj, tuple):
        return tuple([move_to_device(item, device) for item in obj])
    else:
        return obj

def set_seed(seed):
    seed = int(seed)
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.backends.cudnn.enabled = True  # Is this necessary?
        torch.cuda.manual_seed(seed)
<|endoftext|>


<commit_before>import feedparser, json, urllib, math
try: from urllib.parse import urlencode, urlunsplit
except ImportError:
		from urlparse import urlunsplit
		from urllib import urlencode
from functools import partial
from bottle import Request, Response, HTTPResponse, error, route, run, view, template, install, redirect, hook, request, response, abort, static_file, JSONPlugin
from models import *
from mimerender import *

class CustomJsonEncoder(json.JSONEncoder):
	def default(self, obj):
		if isinstance(obj, datetime):
			return str(obj.strftime("%Y-%m-%d %H:%M:%S"))
		if isinstance(obj, Model):
			return obj.__dict__['_data']
		return json.JSONEncoder.default(self, obj)

install(JSONPlugin(json_dumps=lambda s: json.dumps(s, cls=CustomJsonEncoder)))

def is_active(url):
	params = request.query
	valid_keys = ('starred')
	valid_params = dict((k,v) for k, v in params.items() if k in valid_keys)
	fullpath = urlunsplit(('', '', request.path, urlencode(valid_params), ''))
	return 'active' if fullpath == url else ''

mimerender = BottleMimeRender(global_charset = 'utf8')

render_json = lambda **args: json.dumps(args, cls=CustomJsonEncoder)
render_html = lambda tpl='index', **args: lambda **args: template(tpl, channels = Channel.select(), is_active = is_active, **args)

@hook('before_request')
def connect():
	db.connect()

@hook('after_request')
def disconnect():
	db.close()

@route('/')
def index():
	redirect('/items')

@route('/channels/<id:int>/items', method = 'GET')
@route('/items', method = 'GET')
@mimerender(json = render_json, html = render_html('index'))
def items(id = None):
	valid_params = {'1' : True, '0' : False}
	starred = valid_params.get(request.query.getone('starred'))
	read = valid_params.get(request.query.getone('read'))

	channel = request.query.channel or id
	since_id  = request.query.since_id
	max_id = request.query.max_id
	count = int(request.query.count) if request.query.count else 25
	page = int(request.query.page) if request.query.page else 1

	query = Item.select()
	if channel: query = query.where(Item.channel == channel)
	if starred: query = query.where(Item.starred == starred)
	if read: query = query.where(Item.read == read)
	if since_id: query = query.where(Item.id >= since_id)
	if max_id: query = query.where(Item.id <= max_id)
	total_count = query.count()
	if page and count: query = query.paginate(page, count)

	out = { 'items' : list(query.order_by(Item.updated.desc()).limit(count))}

	params = request.query
	params['page'] = page + 1
	out['next'] = urlunsplit(('', '', request.path, urlencode(params), '')) if page <= math.ceil(total_count / count) else None
	params['page'] = page - 1 if page > 1 else 1
	out['prev'] = urlunsplit(('', '', request.path, urlencode(params), '')) if page > 1 else None

	return out

@route('/items/<id:int>', method = 'GET')
def item(id):
	try:
		item = Item.get(Item.id == id)
	except Item.DoesNotExist:
		abort(404, 'Item does not exist')
	return {'item' : item}

@route('/items/<id:int>', method = 'PATCH')
def patch_item(id):
	try:
		item = Item.get(Item.id == id)
	except Item.DoesNotExist:
		abort(404)

	valid_keys = ['read', 'starred']
	for key in set(valid_keys).intersection(set(request.json.keys())):
		setattr(item, key, request.json[key])

	item.save()
	return response.status

@route("/channels", method = 'GET')
def channels():
	return { 'channels' : Channel.select() }

@route("/channels/<id:int>", method = 'GET')
def channel(id):
	try:
		channel = Channel.get(Channel.id == id)
	except:
		abort(404, 'Channel does not exist')
	return { 'channel' : channel }

@route('/channels/<id:int>/delete', method = 'GET')
def delete_channel_confirm(id):
	try:
		channel = Channel.get(Channel.id == id)
	except Channel.DoesNotExist:
		abort(404)

	return template('delete', channel = channel)

@route('/channels/<id:int>', method = 'DELETE')
@route('/channels/<id:int>/delete', method = 'POST')
def delete_channel(id):
	try:
		c = Channel.get(Channel.id == id)
		Item.delete().where(Item.channel == c).execute()
		Channel.delete().where(Channel.id == id).execute()
	except Channel.DoesNotExist:
		abort(404, 'Channel does not exist')
	redirect('/')

@route('/channels/create', method = 'GET')
def create_channel():
	return template('create')

@route('/channels', method = 'POST')
def post_channel():
	url = request.forms.get('url')
	Channel.create_from_url(url)
	channel = Channel.get(Channel.url == url)
	channel.update_feed()
	redirect('/channels/' + str(channel.id) + "/items")

@route('/channels/<id:int>/edit', method = 'GET')
def edit_channel(id):
	channel = Channel.get(Channel.id == id)
	return template('edit', channel = channel)

@route('/channels/<id:int>/edit', method = 'POST')
def edit_channel_post(id):
	title = request.forms.get('title')
	url = request.forms.get('url')
	channel = Channel.get(Channel.id == id)
	channel.title = title
	channel.url = url
	channel.save()
	redirect('/channels/' + str(channel.id) + "/items")

@route('/channels/update', method = 'GET')
def update_channels():
	for c in Channel.select():
		c.update_feed()
	return redirect('/items')

@route('/channels/<id:int>/update', method='GET')
def update_channel(id):
	try:
		c = Channel.get(Channel.id == id)
		c.update_feed()
	except Channel.DoesNotExist:
		abort(404, 'Channel does not exist')
	return redirect('/channels/' + str(c.id) + '/items')

@route('/bla')
def bla():
	return {'hello' : 'bla'}

@error(404)
def error404(error):
	if (request.get_header('Accept') == 'application/json'):
		return Response(json.dumps({'message' : error.body}), status = error.status_code)
	else:
		return Response(error.status + error.body, status = error.status_code)

@route('/static/<filename>')
def server_static(filename):
	return static_file(filename, root='static/')

@route('/favicon.ico')
def get_favicon():
    return server_static('favicon.ico')

if __name__ == '__main__':
	try:
		from mod_wsgi import version
	except:
		run(host='0.0.0.0', port=3000, reloader = True, debug = True)
<commit_msg>Custom error pages.<commit_after>import feedparser, json, urllib, math
try: from urllib.parse import urlencode, urlunsplit
except ImportError:
		from urlparse import urlunsplit
		from urllib import urlencode
from functools import partial
from bottle import  Bottle, Request, Response, HTTPResponse, error, route, run, view, template, install, redirect, hook, request, response, abort, static_file, JSONPlugin
from models import *
from mimerender import *

@error(500)
@error(404)
@error(403)
@error(510)
def custom_error(error):
	if (request.get_header('Accept') == 'application/json'):
		return Response(json.dumps({'message' : error.body}), status = error.status_code)
	else:
		return Response(error.status + ", " + error.body, status = error.status_code)

class CustomJsonEncoder(json.JSONEncoder):
	def default(self, obj):
		if isinstance(obj, datetime):
			return str(obj.strftime("%Y-%m-%d %H:%M:%S"))
		if isinstance(obj, Model):
			return obj.__dict__['_data']
		return json.JSONEncoder.default(self, obj)

install(JSONPlugin(json_dumps=lambda s: json.dumps(s, cls=CustomJsonEncoder)))

def is_active(url):
	params = request.query
	valid_keys = ('starred')
	valid_params = dict((k,v) for k, v in params.items() if k in valid_keys)
	fullpath = urlunsplit(('', '', request.path, urlencode(valid_params), ''))
	return 'active' if fullpath == url else ''

mimerender = BottleMimeRender(global_charset = 'utf8')

render_json = lambda **args: json.dumps(args, cls=CustomJsonEncoder)
render_html = lambda tpl='index', **args: lambda **args: template(tpl, channels = Channel.select(), is_active = is_active, **args)

@hook('before_request')
def connect():
	db.connect()

@hook('after_request')
def disconnect():
	db.close()

@route('/')
def index():
	redirect('/items')

@route('/channels/<id:int>/items', method = 'GET')
@route('/items', method = 'GET')
@mimerender(json = render_json, html = render_html('index'))
def items(id = None):
	valid_params = {'1' : True, '0' : False}
	starred = valid_params.get(request.query.getone('starred'))
	read = valid_params.get(request.query.getone('read'))

	channel = request.query.channel or id
	since_id  = request.query.since_id
	max_id = request.query.max_id
	count = int(request.query.count) if request.query.count else 25
	page = int(request.query.page) if request.query.page else 1

	query = Item.select()
	if channel: query = query.where(Item.channel == channel)
	if starred: query = query.where(Item.starred == starred)
	if read: query = query.where(Item.read == read)
	if since_id: query = query.where(Item.id >= since_id)
	if max_id: query = query.where(Item.id <= max_id)
	total_count = query.count()
	if page and count: query = query.paginate(page, count)

	out = { 'items' : list(query.order_by(Item.updated.desc()).limit(count))}

	params = request.query
	params['page'] = page + 1
	out['next'] = urlunsplit(('', '', request.path, urlencode(params), '')) if page <= math.ceil(total_count / count) else None
	params['page'] = page - 1 if page > 1 else 1
	out['prev'] = urlunsplit(('', '', request.path, urlencode(params), '')) if page > 1 else None

	return out

@route('/items/<id:int>', method = 'GET')
def item(id):
	try:
		item = Item.get(Item.id == id)
	except Item.DoesNotExist:
		abort(404, 'Item does not exist')
	return {'item' : item}

@route('/items/<id:int>', method = 'PATCH')
def patch_item(id):
	try:
		item = Item.get(Item.id == id)
	except Item.DoesNotExist:
		abort(404)

	valid_keys = ['read', 'starred']
	for key in set(valid_keys).intersection(set(request.json.keys())):
		setattr(item, key, request.json[key])

	item.save()
	return response.status

@route("/channels", method = 'GET')
def channels():
	return { 'channels' : Channel.select() }

@route("/channels/<id:int>", method = 'GET')
def channel(id):
	try:
		channel = Channel.get(Channel.id == id)
	except:
		abort(404, 'Channel does not exist')
	return { 'channel' : channel }

@route('/channels/<id:int>/delete', method = 'GET')
def delete_channel_confirm(id):
	try:
		channel = Channel.get(Channel.id == id)
	except Channel.DoesNotExist:
		abort(404)

	return template('delete', channel = channel)

@route('/channels/<id:int>', method = 'DELETE')
@route('/channels/<id:int>/delete', method = 'POST')
def delete_channel(id):
	try:
		c = Channel.get(Channel.id == id)
		Item.delete().where(Item.channel == c).execute()
		Channel.delete().where(Channel.id == id).execute()
	except Channel.DoesNotExist:
		abort(404, 'Channel does not exist')
	redirect('/')

@route('/channels/create', method = 'GET')
def create_channel():
	return template('create')

@route('/channels', method = 'POST')
def post_channel():
	url = request.forms.get('url')
	Channel.create_from_url(url)
	channel = Channel.get(Channel.url == url)
	channel.update_feed()
	redirect('/channels/' + str(channel.id) + "/items")

@route('/channels/<id:int>/edit', method = 'GET')
def edit_channel(id):
	channel = Channel.get(Channel.id == id)
	return template('edit', channel = channel)

@route('/channels/<id:int>/edit', method = 'POST')
def edit_channel_post(id):
	title = request.forms.get('title')
	url = request.forms.get('url')
	channel = Channel.get(Channel.id == id)
	channel.title = title
	channel.url = url
	channel.save()
	redirect('/channels/' + str(channel.id) + "/items")

@route('/channels/update', method = 'GET')
def update_channels():
	for c in Channel.select():
		c.update_feed()
	return redirect('/items')

@route('/channels/<id:int>/update', method='GET')
def update_channel(id):
	try:
		c = Channel.get(Channel.id == id)
		c.update_feed()
	except Channel.DoesNotExist:
		abort(404, 'Channel does not exist')
	return redirect('/channels/' + str(c.id) + '/items')

@route('/static/<filename>')
def server_static(filename):
	return static_file(filename, root='static/')

@route('/favicon.ico')
def get_favicon():
    return server_static('favicon.ico')

if __name__ == '__main__':
	try:
		from mod_wsgi import version
	except:
		run(host='0.0.0.0', port=3000, reloader = True, debug = True)
<|endoftext|>


<commit_before>#!/usr/bin/env bash

# This will be the userdata for Linux EC2 images to run SystemPrep via WatchMaker.<commit_msg>Fixing Python placeholder with newline and the right executable in first line.<commit_after>#!/usr/bin/env python

# This will be the userdata for Linux EC2 images to run SystemPrep via WatchMaker.
<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

from UM.Signal import Signal, signalemitter
from . import USBPrinterOutputDevice
from UM.Application import Application
from UM.Resources import Resources
from UM.Logger import Logger
from UM.PluginRegistry import PluginRegistry
from UM.OutputDevice.OutputDevicePlugin import OutputDevicePlugin
from cura.PrinterOutputDevice import ConnectionState
from UM.Qt.ListModel import ListModel
from UM.Message import Message

from cura.CuraApplication import CuraApplication

import threading
import platform
import glob
import time
import os.path
from UM.Extension import Extension

from PyQt5.QtQml import QQmlComponent, QQmlContext
from PyQt5.QtCore import QUrl, QObject, pyqtSlot, pyqtProperty, pyqtSignal, Qt
from UM.i18n import i18nCatalog
i18n_catalog = i18nCatalog("cura")

##  Manager class that ensures that a usbPrinteroutput device is created for every connected USB printer.
@signalemitter
class USBPrinterOutputDeviceManager(QObject, OutputDevicePlugin, Extension):
    def __init__(self, parent = None):
        super().__init__(parent = parent)
        self._serial_port_list = []
        self._usb_output_devices = {}
        self._usb_output_devices_model = None
        self._update_thread = threading.Thread(target = self._updateThread)
        self._update_thread.setDaemon(True)

        self._check_updates = True
        self._firmware_view = None

        Application.getInstance().applicationShuttingDown.connect(self.stop)
        self.addUSBOutputDeviceSignal.connect(self.addOutputDevice) #Because the model needs to be created in the same thread as the QMLEngine, we use a signal.

    addUSBOutputDeviceSignal = Signal()
    connectionStateChanged = pyqtSignal()

    progressChanged = pyqtSignal()
    firmwareUpdateChange = pyqtSignal()

    @pyqtProperty(float, notify = progressChanged)
    def progress(self):
        progress = 0
        for printer_name, device in self._usb_output_devices.items(): # TODO: @UnusedVariable "printer_name"
            progress += device.progress
        return progress / len(self._usb_output_devices)

    @pyqtProperty(int, notify = progressChanged)
    def errorCode(self):
        for printer_name, device in self._usb_output_devices.items(): # TODO: @UnusedVariable "printer_name"
            if device._error_code:
                return device._error_code
        return 0

    ##  Return True if all printers finished firmware update
    @pyqtProperty(float, notify = firmwareUpdateChange)
    def firmwareUpdateCompleteStatus(self):
        complete = True
        for printer_name, device in self._usb_output_devices.items(): # TODO: @UnusedVariable "printer_name"
            if not device.firmwareUpdateFinished:
                complete = False
        return complete

    def start(self):
        self._check_updates = True
        self._update_thread.start()

    def stop(self):
        self._check_updates = False
        try:
            self._update_thread.join()
        except RuntimeError:
            pass

    def _updateThread(self):
        while self._check_updates:
            result = self.getSerialPortList(only_list_usb = True)
            self._addRemovePorts(result)
            time.sleep(5)

    ##  Show firmware interface.
    #   This will create the view if its not already created.
    def spawnFirmwareInterface(self, serial_port):
        if self._firmware_view is None:
            path = QUrl.fromLocalFile(os.path.join(PluginRegistry.getInstance().getPluginPath("USBPrinting"), "FirmwareUpdateWindow.qml"))
            component = QQmlComponent(Application.getInstance()._engine, path)

            self._firmware_context = QQmlContext(Application.getInstance()._engine.rootContext())
            self._firmware_context.setContextProperty("manager", self)
            self._firmware_view = component.create(self._firmware_context)

        self._firmware_view.show()

    @pyqtSlot(str)
    def updateAllFirmware(self, file_name):
        if file_name.startswith("file://"):
            file_name = QUrl(file_name).toLocalFile() # File dialogs prepend the path with file://, which we don't need / want
        if not self._usb_output_devices:
            Message(i18n_catalog.i18nc("@info", "Unable to update firmware because there are no printers connected.")).show()
            return

        for printer_connection in self._usb_output_devices:
            self._usb_output_devices[printer_connection].resetFirmwareUpdate()
        self.spawnFirmwareInterface("")
        for printer_connection in self._usb_output_devices:
            try:
                self._usb_output_devices[printer_connection].updateFirmware(file_name)
            except FileNotFoundError:
                # Should only happen in dev environments where the resources/firmware folder is absent.
                self._usb_output_devices[printer_connection].setProgress(100, 100)
                Logger.log("w", "No firmware found for printer %s called '%s'", printer_connection, file_name)
                Message(i18n_catalog.i18nc("@info",
                    "Could not find firmware required for the printer at %s.") % printer_connection).show()
                self._firmware_view.close()

                continue

    @pyqtSlot(str, str, result = bool)
    def updateFirmwareBySerial(self, serial_port, file_name):
        if serial_port in self._usb_output_devices:
            self.spawnFirmwareInterface(self._usb_output_devices[serial_port].getSerialPort())
            try:
                self._usb_output_devices[serial_port].updateFirmware(file_name)
            except FileNotFoundError:
                self._firmware_view.close()
                Logger.log("e", "Could not find firmware required for this machine called '%s'", file_name)
                return False
            return True
        return False

    ##  Return the singleton instance of the USBPrinterManager
    @classmethod
    def getInstance(cls, engine = None, script_engine = None):
        # Note: Explicit use of class name to prevent issues with inheritance.
        if USBPrinterOutputDeviceManager._instance is None:
            USBPrinterOutputDeviceManager._instance = cls()

        return USBPrinterOutputDeviceManager._instance

    @pyqtSlot(result = str)
    def getDefaultFirmwareName(self):
        # Check if there is a valid global container stack
        global_container_stack = Application.getInstance().getGlobalContainerStack()
        if not global_container_stack:
            Logger.log("e", "There is no global container stack. Can not update firmware.")
            self._firmware_view.close()
            return ""

        # The bottom of the containerstack is the machine definition
        machine_id = global_container_stack.getBottom().id

        machine_has_heated_bed = global_container_stack.getProperty("machine_heated_bed", "value")

        if platform.system() == "Linux":
            baudrate = 115200
        else:
            baudrate = 250000

        # NOTE: The keyword used here is the id of the machine. You can find the id of your machine in the *.json file, eg.
        # https://github.com/Ultimaker/Cura/blob/master/resources/machines/ultimaker_original.json#L2
        # The *.hex files are stored at a seperate repository:
        # https://github.com/Ultimaker/cura-binary-data/tree/master/cura/resources/firmware
        machine_without_extras  = {"bq_witbox"                : "MarlinWitbox.hex",
                                   "bq_hephestos_2"           : "MarlinHephestos2.hex",
                                   "ultimaker_original"       : "MarlinUltimaker-{baudrate}.hex",
                                   "ultimaker_original_plus"  : "MarlinUltimaker-UMOP-{baudrate}.hex",
                                   "ultimaker_original_dual"  : "MarlinUltimaker-{baudrate}-dual.hex",
                                   "ultimaker2"               : "MarlinUltimaker2.hex",
                                   "ultimaker2_go"            : "MarlinUltimaker2go.hex",
                                   "ultimaker2_plus"          : "MarlinUltimaker2plus.hex",
                                   "ultimaker2_extended"      : "MarlinUltimaker2extended.hex",
                                   "ultimaker2_extended_plus" : "MarlinUltimaker2extended-plus.hex",
                                   }
        machine_with_heated_bed = {"ultimaker_original"       : "MarlinUltimaker-HBK-{baudrate}.hex",
                                   "ultimaker_original_dual"  : "MarlinUltimaker-HBK-{baudrate}-dual.hex",
                                   }
        ##TODO: Add check for multiple extruders
        hex_file = None
        if machine_id in machine_without_extras.keys():  # The machine needs to be defined here!
            if machine_id in machine_with_heated_bed.keys() and machine_has_heated_bed:
                Logger.log("d", "Choosing firmware with heated bed enabled for machine %s.", machine_id)
                hex_file = machine_with_heated_bed[machine_id]  # Return firmware with heated bed enabled
            else:
                Logger.log("d", "Choosing basic firmware for machine %s.", machine_id)
                hex_file = machine_without_extras[machine_id]  # Return "basic" firmware
        else:
            Logger.log("w", "There is no firmware for machine %s.", machine_id)

        if hex_file:
            return Resources.getPath(CuraApplication.ResourceTypes.Firmware, hex_file.format(baudrate=baudrate))
        else:
            Logger.log("w", "Could not find any firmware for machine %s.", machine_id)
            return ""

    ##  Helper to identify serial ports (and scan for them)
    def _addRemovePorts(self, serial_ports):
        # First, find and add all new or changed keys
        for serial_port in list(serial_ports):
            if serial_port not in self._serial_port_list:
                self.addUSBOutputDeviceSignal.emit(serial_port)  # Hack to ensure its created in main thread
                continue
        self._serial_port_list = list(serial_ports)

        devices_to_remove = []
        for port, device in self._usb_output_devices.items():
            if port not in self._serial_port_list:
                device.close()
                devices_to_remove.append(port)

        for port in devices_to_remove:
            del self._usb_output_devices[port]

    ##  Because the model needs to be created in the same thread as the QMLEngine, we use a signal.
    def addOutputDevice(self, serial_port):
        device = USBPrinterOutputDevice.USBPrinterOutputDevice(serial_port)
        device.connectionStateChanged.connect(self._onConnectionStateChanged)
        device.connect()
        device.progressChanged.connect(self.progressChanged)
        device.firmwareUpdateChange.connect(self.firmwareUpdateChange)
        self._usb_output_devices[serial_port] = device

    ##  If one of the states of the connected devices change, we might need to add / remove them from the global list.
    def _onConnectionStateChanged(self, serial_port):
        try:
            if self._usb_output_devices[serial_port].connectionState == ConnectionState.connected:
                self.getOutputDeviceManager().addOutputDevice(self._usb_output_devices[serial_port])
            else:
                self.getOutputDeviceManager().removeOutputDevice(serial_port)
            self.connectionStateChanged.emit()
        except KeyError:
            pass  # no output device by this device_id found in connection list.

    @pyqtProperty(QObject , notify = connectionStateChanged)
    def connectedPrinterList(self):
        self._usb_output_devices_model = ListModel()
        self._usb_output_devices_model.addRoleName(Qt.UserRole + 1, "name")
        self._usb_output_devices_model.addRoleName(Qt.UserRole + 2, "printer")
        for connection in self._usb_output_devices:
            if self._usb_output_devices[connection].connectionState == ConnectionState.connected:
                self._usb_output_devices_model.appendItem({"name": connection, "printer": self._usb_output_devices[connection]})
        return self._usb_output_devices_model

    ##  Create a list of serial ports on the system.
    #   \param only_list_usb If true, only usb ports are listed
    def getSerialPortList(self, only_list_usb = False):
        base_list = []
        if platform.system() == "Windows":
            import winreg #@UnresolvedImport
            try:
                key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,"HARDWARE\\DEVICEMAP\\SERIALCOMM")
                i = 0
                while True:
                    values = winreg.EnumValue(key, i)
                    if not only_list_usb or "USBSER" in values[0]:
                        base_list += [values[1]]
                    i += 1
            except Exception as e:
                pass
        else:
            if only_list_usb:
                base_list = base_list + glob.glob("/dev/ttyUSB*") + glob.glob("/dev/ttyACM*") + glob.glob("/dev/cu.usb*")
                base_list = filter(lambda s: "Bluetooth" not in s, base_list) # Filter because mac sometimes puts them in the list
            else:
                base_list = base_list + glob.glob("/dev/ttyUSB*") + glob.glob("/dev/ttyACM*") + glob.glob("/dev/cu.*") + glob.glob("/dev/tty.usb*") + glob.glob("/dev/rfcomm*") + glob.glob("/dev/serial/by-id/*")
        return list(base_list)

    _instance = None
<commit_msg>Calling the stop of USBPrinterOutput device no longer joins the thread.<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Cura is released under the terms of the AGPLv3 or higher.

from UM.Signal import Signal, signalemitter
from . import USBPrinterOutputDevice
from UM.Application import Application
from UM.Resources import Resources
from UM.Logger import Logger
from UM.PluginRegistry import PluginRegistry
from UM.OutputDevice.OutputDevicePlugin import OutputDevicePlugin
from cura.PrinterOutputDevice import ConnectionState
from UM.Qt.ListModel import ListModel
from UM.Message import Message

from cura.CuraApplication import CuraApplication

import threading
import platform
import glob
import time
import os.path
from UM.Extension import Extension

from PyQt5.QtQml import QQmlComponent, QQmlContext
from PyQt5.QtCore import QUrl, QObject, pyqtSlot, pyqtProperty, pyqtSignal, Qt
from UM.i18n import i18nCatalog
i18n_catalog = i18nCatalog("cura")

##  Manager class that ensures that a usbPrinteroutput device is created for every connected USB printer.
@signalemitter
class USBPrinterOutputDeviceManager(QObject, OutputDevicePlugin, Extension):
    def __init__(self, parent = None):
        super().__init__(parent = parent)
        self._serial_port_list = []
        self._usb_output_devices = {}
        self._usb_output_devices_model = None
        self._update_thread = threading.Thread(target = self._updateThread)
        self._update_thread.setDaemon(True)

        self._check_updates = True
        self._firmware_view = None

        Application.getInstance().applicationShuttingDown.connect(self.stop)
        self.addUSBOutputDeviceSignal.connect(self.addOutputDevice) #Because the model needs to be created in the same thread as the QMLEngine, we use a signal.

    addUSBOutputDeviceSignal = Signal()
    connectionStateChanged = pyqtSignal()

    progressChanged = pyqtSignal()
    firmwareUpdateChange = pyqtSignal()

    @pyqtProperty(float, notify = progressChanged)
    def progress(self):
        progress = 0
        for printer_name, device in self._usb_output_devices.items(): # TODO: @UnusedVariable "printer_name"
            progress += device.progress
        return progress / len(self._usb_output_devices)

    @pyqtProperty(int, notify = progressChanged)
    def errorCode(self):
        for printer_name, device in self._usb_output_devices.items(): # TODO: @UnusedVariable "printer_name"
            if device._error_code:
                return device._error_code
        return 0

    ##  Return True if all printers finished firmware update
    @pyqtProperty(float, notify = firmwareUpdateChange)
    def firmwareUpdateCompleteStatus(self):
        complete = True
        for printer_name, device in self._usb_output_devices.items(): # TODO: @UnusedVariable "printer_name"
            if not device.firmwareUpdateFinished:
                complete = False
        return complete

    def start(self):
        self._check_updates = True
        self._update_thread.start()

    def stop(self):
        self._check_updates = False

    def _updateThread(self):
        while self._check_updates:
            result = self.getSerialPortList(only_list_usb = True)
            self._addRemovePorts(result)
            time.sleep(5)

    ##  Show firmware interface.
    #   This will create the view if its not already created.
    def spawnFirmwareInterface(self, serial_port):
        if self._firmware_view is None:
            path = QUrl.fromLocalFile(os.path.join(PluginRegistry.getInstance().getPluginPath("USBPrinting"), "FirmwareUpdateWindow.qml"))
            component = QQmlComponent(Application.getInstance()._engine, path)

            self._firmware_context = QQmlContext(Application.getInstance()._engine.rootContext())
            self._firmware_context.setContextProperty("manager", self)
            self._firmware_view = component.create(self._firmware_context)

        self._firmware_view.show()

    @pyqtSlot(str)
    def updateAllFirmware(self, file_name):
        if file_name.startswith("file://"):
            file_name = QUrl(file_name).toLocalFile() # File dialogs prepend the path with file://, which we don't need / want
        if not self._usb_output_devices:
            Message(i18n_catalog.i18nc("@info", "Unable to update firmware because there are no printers connected.")).show()
            return

        for printer_connection in self._usb_output_devices:
            self._usb_output_devices[printer_connection].resetFirmwareUpdate()
        self.spawnFirmwareInterface("")
        for printer_connection in self._usb_output_devices:
            try:
                self._usb_output_devices[printer_connection].updateFirmware(file_name)
            except FileNotFoundError:
                # Should only happen in dev environments where the resources/firmware folder is absent.
                self._usb_output_devices[printer_connection].setProgress(100, 100)
                Logger.log("w", "No firmware found for printer %s called '%s'", printer_connection, file_name)
                Message(i18n_catalog.i18nc("@info",
                    "Could not find firmware required for the printer at %s.") % printer_connection).show()
                self._firmware_view.close()

                continue

    @pyqtSlot(str, str, result = bool)
    def updateFirmwareBySerial(self, serial_port, file_name):
        if serial_port in self._usb_output_devices:
            self.spawnFirmwareInterface(self._usb_output_devices[serial_port].getSerialPort())
            try:
                self._usb_output_devices[serial_port].updateFirmware(file_name)
            except FileNotFoundError:
                self._firmware_view.close()
                Logger.log("e", "Could not find firmware required for this machine called '%s'", file_name)
                return False
            return True
        return False

    ##  Return the singleton instance of the USBPrinterManager
    @classmethod
    def getInstance(cls, engine = None, script_engine = None):
        # Note: Explicit use of class name to prevent issues with inheritance.
        if USBPrinterOutputDeviceManager._instance is None:
            USBPrinterOutputDeviceManager._instance = cls()

        return USBPrinterOutputDeviceManager._instance

    @pyqtSlot(result = str)
    def getDefaultFirmwareName(self):
        # Check if there is a valid global container stack
        global_container_stack = Application.getInstance().getGlobalContainerStack()
        if not global_container_stack:
            Logger.log("e", "There is no global container stack. Can not update firmware.")
            self._firmware_view.close()
            return ""

        # The bottom of the containerstack is the machine definition
        machine_id = global_container_stack.getBottom().id

        machine_has_heated_bed = global_container_stack.getProperty("machine_heated_bed", "value")

        if platform.system() == "Linux":
            baudrate = 115200
        else:
            baudrate = 250000

        # NOTE: The keyword used here is the id of the machine. You can find the id of your machine in the *.json file, eg.
        # https://github.com/Ultimaker/Cura/blob/master/resources/machines/ultimaker_original.json#L2
        # The *.hex files are stored at a seperate repository:
        # https://github.com/Ultimaker/cura-binary-data/tree/master/cura/resources/firmware
        machine_without_extras  = {"bq_witbox"                : "MarlinWitbox.hex",
                                   "bq_hephestos_2"           : "MarlinHephestos2.hex",
                                   "ultimaker_original"       : "MarlinUltimaker-{baudrate}.hex",
                                   "ultimaker_original_plus"  : "MarlinUltimaker-UMOP-{baudrate}.hex",
                                   "ultimaker_original_dual"  : "MarlinUltimaker-{baudrate}-dual.hex",
                                   "ultimaker2"               : "MarlinUltimaker2.hex",
                                   "ultimaker2_go"            : "MarlinUltimaker2go.hex",
                                   "ultimaker2_plus"          : "MarlinUltimaker2plus.hex",
                                   "ultimaker2_extended"      : "MarlinUltimaker2extended.hex",
                                   "ultimaker2_extended_plus" : "MarlinUltimaker2extended-plus.hex",
                                   }
        machine_with_heated_bed = {"ultimaker_original"       : "MarlinUltimaker-HBK-{baudrate}.hex",
                                   "ultimaker_original_dual"  : "MarlinUltimaker-HBK-{baudrate}-dual.hex",
                                   }
        ##TODO: Add check for multiple extruders
        hex_file = None
        if machine_id in machine_without_extras.keys():  # The machine needs to be defined here!
            if machine_id in machine_with_heated_bed.keys() and machine_has_heated_bed:
                Logger.log("d", "Choosing firmware with heated bed enabled for machine %s.", machine_id)
                hex_file = machine_with_heated_bed[machine_id]  # Return firmware with heated bed enabled
            else:
                Logger.log("d", "Choosing basic firmware for machine %s.", machine_id)
                hex_file = machine_without_extras[machine_id]  # Return "basic" firmware
        else:
            Logger.log("w", "There is no firmware for machine %s.", machine_id)

        if hex_file:
            return Resources.getPath(CuraApplication.ResourceTypes.Firmware, hex_file.format(baudrate=baudrate))
        else:
            Logger.log("w", "Could not find any firmware for machine %s.", machine_id)
            return ""

    ##  Helper to identify serial ports (and scan for them)
    def _addRemovePorts(self, serial_ports):
        # First, find and add all new or changed keys
        for serial_port in list(serial_ports):
            if serial_port not in self._serial_port_list:
                self.addUSBOutputDeviceSignal.emit(serial_port)  # Hack to ensure its created in main thread
                continue
        self._serial_port_list = list(serial_ports)

        devices_to_remove = []
        for port, device in self._usb_output_devices.items():
            if port not in self._serial_port_list:
                device.close()
                devices_to_remove.append(port)

        for port in devices_to_remove:
            del self._usb_output_devices[port]

    ##  Because the model needs to be created in the same thread as the QMLEngine, we use a signal.
    def addOutputDevice(self, serial_port):
        device = USBPrinterOutputDevice.USBPrinterOutputDevice(serial_port)
        device.connectionStateChanged.connect(self._onConnectionStateChanged)
        device.connect()
        device.progressChanged.connect(self.progressChanged)
        device.firmwareUpdateChange.connect(self.firmwareUpdateChange)
        self._usb_output_devices[serial_port] = device

    ##  If one of the states of the connected devices change, we might need to add / remove them from the global list.
    def _onConnectionStateChanged(self, serial_port):
        try:
            if self._usb_output_devices[serial_port].connectionState == ConnectionState.connected:
                self.getOutputDeviceManager().addOutputDevice(self._usb_output_devices[serial_port])
            else:
                self.getOutputDeviceManager().removeOutputDevice(serial_port)
            self.connectionStateChanged.emit()
        except KeyError:
            pass  # no output device by this device_id found in connection list.

    @pyqtProperty(QObject , notify = connectionStateChanged)
    def connectedPrinterList(self):
        self._usb_output_devices_model = ListModel()
        self._usb_output_devices_model.addRoleName(Qt.UserRole + 1, "name")
        self._usb_output_devices_model.addRoleName(Qt.UserRole + 2, "printer")
        for connection in self._usb_output_devices:
            if self._usb_output_devices[connection].connectionState == ConnectionState.connected:
                self._usb_output_devices_model.appendItem({"name": connection, "printer": self._usb_output_devices[connection]})
        return self._usb_output_devices_model

    ##  Create a list of serial ports on the system.
    #   \param only_list_usb If true, only usb ports are listed
    def getSerialPortList(self, only_list_usb = False):
        base_list = []
        if platform.system() == "Windows":
            import winreg #@UnresolvedImport
            try:
                key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE,"HARDWARE\\DEVICEMAP\\SERIALCOMM")
                i = 0
                while True:
                    values = winreg.EnumValue(key, i)
                    if not only_list_usb or "USBSER" in values[0]:
                        base_list += [values[1]]
                    i += 1
            except Exception as e:
                pass
        else:
            if only_list_usb:
                base_list = base_list + glob.glob("/dev/ttyUSB*") + glob.glob("/dev/ttyACM*") + glob.glob("/dev/cu.usb*")
                base_list = filter(lambda s: "Bluetooth" not in s, base_list) # Filter because mac sometimes puts them in the list
            else:
                base_list = base_list + glob.glob("/dev/ttyUSB*") + glob.glob("/dev/ttyACM*") + glob.glob("/dev/cu.*") + glob.glob("/dev/tty.usb*") + glob.glob("/dev/rfcomm*") + glob.glob("/dev/serial/by-id/*")
        return list(base_list)

    _instance = None
<|endoftext|>