<commit_before>#!/usr/bin/env python3

from __future__ import print_function

import argparse
import datetime as dt
import json
import os
import shlex
import smtplib
import subprocess
import sys
from email.mime.text import MIMEText

import feedparser

# python 2 and 3 do different things with urllib
try:
    from urllib.request import urlopen
except ImportError:
    from urllib2 import urlopen

class Paper:
    """a Paper is a single paper listed on arXiv.  In addition to the
       paper's title, ID, and URL (obtained from arXiv), we also store
       which keywords it matched and which Slack channel it should go
       to"""

    def __init__(self, arxiv_id, title, url, keywords, channels):
        self.arxiv_id = arxiv_id
        self.title = title.replace("'", r"")
        self.url = url
        self.keywords = list(keywords)
        self.channels = list(set(channels))
        self.posted_to_slack = 0

    def __str__(self):
        t = " ".join(self.title.split())  # remove extra spaces
        return u"{} : {}\n  {}\n".format(self.arxiv_id, t, self.url)

    def kw_str(self):
        """ return the union of keywords """
        return ", ".join(self.keywords)

    def __lt__(self, other):
        """we compare Papers by the number of keywords, and then
           alphabetically by the union of their keywords"""

        if len(self.keywords) == len(other.keywords):
            return self.kw_str() < other.kw_str()

        return len(self.keywords) < len(other.keywords)

class Keyword:
    """a Keyword includes: the text we should match, how the matching
       should be done (unique or any), which words, if present, negate
       the match, and what Slack channel this keyword is associated with"""

    def __init__(self, name, matching="any", channel=None, excludes=None):
        self.name = name
        self.matching = matching
        self.channel = channel
        self.excludes = list(set(excludes))

    def __str__(self):
        return "{}: matching={}, channel={}, NOTs={}".format(
            self.name, self.matching, self.channel, self.excludes)

class AstrophQuery:
    """ a class to define a query to the arXiv astroph papers """

    def __init__(self, start_date, end_date, max_papers, old_id=None):
        self.start_date = start_date
        self.end_date = end_date
        self.max_papers = max_papers
        self.old_id = old_id

        self.base_url = "http://export.arxiv.org/api/query?"
        self.sort_query = "max_results={}&sortBy=submittedDate&sortOrder=descending".format(
            self.max_papers)

        self.subcat = ["GA", "CO", "EP", "HE", "IM", "SR"]

    def get_cat_query(self):
        """ create the category portion of the astro ph query """

        cat_query = "%28"  # open parenthesis
        for n, s in enumerate(self.subcat):
            cat_query += "astro-ph.{}".format(s)
            if n < len(self.subcat)-1:
                cat_query += "+OR+"
            else:
                cat_query += "%29"  # close parenthesis

        return cat_query

    def get_range_query(self):
        """ get the query string for the date range """

        # here the 2000 on each date is 8:00pm
        range_str = "[{}2000+TO+{}2000]".format(self.start_date.strftime("%Y%m%d"),
                                                self.end_date.strftime("%Y%m%d"))
        range_query = "lastUpdatedDate:{}".format(range_str)
        return range_query

    def get_url(self):
        """ create the URL we will use to query arXiv """

        cat_query = self.get_cat_query()
        range_query = self.get_range_query()

        full_query = "search_query={}+AND+{}&{}".format(cat_query, range_query, self.sort_query)

        return self.base_url + full_query

    def do_query(self, keywords=None, old_id=None):
        """ perform the actual query """

        # note, in python3 this will be bytes not str
        response = urlopen(self.get_url()).read()
        response = response.replace(b"author", b"contributor")

        # this feedparser magic comes from the example of Julius Lucks / Andrea Zonca
        # https://github.com/zonca/python-parse-arxiv/blob/master/python_arXiv_parsing_example.py
        feedparser._FeedParserMixin.namespaces['http://a9.com/-/spec/opensearch/1.1/'] = 'opensearch'
        feedparser._FeedParserMixin.namespaces['http://arxiv.org/schemas/atom'] = 'arxiv'

        feed = feedparser.parse(response)

        if feed.feed.opensearch_totalresults == 0:
            sys.exit("no results found")

        results = []

        latest_id = None

        for e in feed.entries:

            arxiv_id = e.id.split("/abs/")[-1]
            title = e.title.replace("\n", " ")

            # the papers are sorted now such that the first is the
            # most recent -- we want to store this id, so the next
            # time we run the script, we can pick up from here
            if latest_id is None:
                latest_id = arxiv_id

            # now check if we hit the old_id -- this is where we
            # left off last time.  Note things may not be in id order,
            # so we keep looking through the entire list of returned
            # results.
            if old_id is not None:
                if arxiv_id < old_id:
                    continue

            # link
            for l in e.links:
                if l.rel == "alternate":
                    url = l.href

            abstract = e.summary

            # any keyword matches?
            # we do two types of matches here.  If the keyword tuple has the "any"
            # qualifier, then we don't care how it appears in the text, but if
            # it has "unique", then we want to make sure only that word matches,
            # i.e., "nova" and not "supernova".  If any of the exclude words associated
            # with the keyword are present, then we reject any match
            keys_matched = []
            channels = []
            for k in keywords:
                # first check the "NOT"s
                excluded = False
                for n in k.excludes:
                    if n in abstract.lower().replace("\n", " ") or n in title.lower():
                        # we've matched one of the excludes
                        excluded = True

                if excluded:
                    continue

                if k.matching == "any":
                    if k.name in abstract.lower().replace("\n", " ") or k.name in title.lower():
                        keys_matched.append(k.name)
                        channels.append(k.channel)

                elif k.matching == "unique":
                    qa = [l.lower().strip('\":.,!?') for l in abstract.split()]
                    qt = [l.lower().strip('\":.,!?') for l in title.split()]
                    if k.name in qa + qt:
                        keys_matched.append(k.name)
                        channels.append(k.channel)

            if keys_matched:
                results.append(Paper(arxiv_id, title, url, keys_matched, channels))

        return results, latest_id

def report(body, subject, sender, receiver):
    """ send an email """
    msg = MIMEText(body)
    msg['Subject'] = subject
    msg['From'] = sender
    msg['To'] = receiver

    try:
        sm = smtplib.SMTP('localhost')
        sm.sendmail(sender, receiver, msg.as_string())
    except smtplib.SMTPException:
        sys.exit("ERROR sending mail")

def search_astroph(keywords, old_id=None):
    """ do the actual search though astro-ph by first querying astro-ph
        for the latest papers and then looking for keyword matches"""

    today = dt.date.today()
    day = dt.timedelta(days=1)

    max_papers = 1000

    # we pick a wide-enough search range to ensure we catch papers
    # if there is a holiday

    # also, something wierd happens -- the arxiv ids appear to be
    # in descending order if you look at the "pastweek" listing
    # but the submission dates can vary wildly.  It seems that some
    # papers are held for a week or more before appearing.
    q = AstrophQuery(today - 10*day, today, max_papers, old_id=old_id)
    print(q.get_url())

    papers, last_id = q.do_query(keywords=keywords, old_id=old_id)

    papers.sort(reverse=True)

    return papers, last_id

def send_email(papers, mail=None):

    # compose the body of our e-mail
    body = ""

    # sort papers by keywords
    current_kw = None
    for p in papers:
        if not p.kw_str() == current_kw:
            current_kw = p.kw_str()
            body += "\nkeywords: {}\n\n".format(current_kw)

        body += u"{}\n".format(p)

    # e-mail it
    if not len(papers) == 0:
        if not mail is None:
            report(body, "astro-ph papers of interest",
                   "lazy-astroph@localhost <lazy-astroph search>", mail)
        else:
            print(body)

def run(string):
    """ run a UNIX command """

    # shlex.split will preserve inner quotes
    prog = shlex.split(string)
    p0 = subprocess.Popen(prog, stdout=subprocess.PIPE,
                          stderr=subprocess.STDOUT)

    stdout0, stderr0 = p0.communicate()
    rc = p0.returncode
    p0.stdout.close()

    return stdout0, stderr0, rc

def slack_post(papers, channel_req, username=None, icon_emoji=None, webhook=None):
    """ post the information to a slack channel """

    # loop by channel
    for c in channel_req:
        channel_body = ""
        for p in papers:
            if not p.posted_to_slack:
                if c in p.channels:
                    if len(p.keywords) >= channel_req[c]:
                        keywds = ", ".join(p.keywords).strip()
                        channel_body += u"{} [{}]\n\n".format(p, keywds)
                        p.posted_to_slack = 1

        if webhook is None:
            print("channel: {}".format(c))
            print(channel_body)
            continue

        payload = {}
        payload["channel"] = c
        if username is not None:
            payload["username"] = username
        if icon_emoji is not None:
            payload["icon_emoji"] = icon_emoji
        payload["text"] = channel_body

        cmd = "curl -X POST --data-urlencode 'payload={}' {}".format(json.dumps(payload), webhook)
        run(cmd)

    sys.exit("done")

def doit():
    """ the main driver for the lazy-astroph script """

    # parse runtime parameters
    parser = argparse.ArgumentParser()

    parser.add_argument("-m", help="e-mail address to send report to",
                        type=str, default=None)
    parser.add_argument("inputs", help="inputs file containing keywords",
                        type=str, nargs=1)
    parser.add_argument("-w", help="file containing slack webhook URL",
                        type=str, default=None)
    parser.add_argument("-u", help="slack username appearing in post",
                        type=str, default=None)
    parser.add_argument("-e", help="slack icon_emoji appearing in post",
                        type=str, default=None)
    parser.add_argument("--dry_run",
                        help="don't send any mail or slack posts and don't update the marker where we left off",
                        action="store_true")
    args = parser.parse_args()

    # get the keywords
    keywords = []
    try:
        f = open(args.inputs[0], "r")
    except:
        sys.exit("ERROR: unable to open inputs file")
    else:
        channel = None
        channel_req = {}
        for line in f:
            l = line.lower().rstrip()

            if l == "":
                continue

            elif l.startswith("#") or l.startswith("@"):
                # this line defines a channel
                ch = l.split()
                channel = ch[0]
                if len(ch) == 2:
                    requires = int(ch[1].split("=")[1])
                else:
                    requires = 1
                channel_req[channel] = requires

            else:
                # this line has a keyword (and optional NOT keywords)
                if "not:" in l:
                    kw, nots = l.split("not:")
                    kw = kw.strip()
                    excludes = [x.strip() for x in nots.split(",")]
                else:
                    kw = l.strip()
                    excludes = []

                if kw[len(kw)-1] == "-":
                    matching = "unique"
                    kw = kw[:len(kw)-1]
                else:
                    matching = "any"

                keywords.append(Keyword(kw, matching=matching,
                                        channel=channel, excludes=excludes))

    # have we done this before? if so, read the .lazy_astroph file to get
    # the id of the paper we left off with
    param_file = os.path.expanduser("~") + "/.lazy_astroph"
    try:
        f = open(param_file, "r")
    except:
        old_id = None
    else:
        old_id = f.readline().rstrip()
        f.close()

    papers, last_id = search_astroph(keywords, old_id=old_id)

    if not args.dry_run:
        send_email(papers, mail=args.m)

        if not args.w is None:
            try:
                f = open(args.w)
            except:
                sys.exit("ERROR: unable to open webhook file")

            webhook = str(f.readline())
            f.close()
        else:
            webhook = None

        slack_post(papers, channel_req, icon_emoji=args.e, username=args.u, webhook=webhook)

        try:
            f = open(param_file, "w")
        except:
            sys.exit("ERROR: unable to open parameter file for writting")
        else:
            f.write(last_id)
            f.close()
    else:
        send_email(papers, mail=None)

if __name__ == "__main__":
    doit()
<commit_msg>use the platform module to get the hostname for the "from"<commit_after>#!/usr/bin/env python3

from __future__ import print_function

import argparse
import datetime as dt
import json
import os
import platform
import shlex
import smtplib
import subprocess
import sys
from email.mime.text import MIMEText

import feedparser

# python 2 and 3 do different things with urllib
try:
    from urllib.request import urlopen
except ImportError:
    from urllib2 import urlopen

class Paper:
    """a Paper is a single paper listed on arXiv.  In addition to the
       paper's title, ID, and URL (obtained from arXiv), we also store
       which keywords it matched and which Slack channel it should go
       to"""

    def __init__(self, arxiv_id, title, url, keywords, channels):
        self.arxiv_id = arxiv_id
        self.title = title.replace("'", r"")
        self.url = url
        self.keywords = list(keywords)
        self.channels = list(set(channels))
        self.posted_to_slack = 0

    def __str__(self):
        t = " ".join(self.title.split())  # remove extra spaces
        return u"{} : {}\n  {}\n".format(self.arxiv_id, t, self.url)

    def kw_str(self):
        """ return the union of keywords """
        return ", ".join(self.keywords)

    def __lt__(self, other):
        """we compare Papers by the number of keywords, and then
           alphabetically by the union of their keywords"""

        if len(self.keywords) == len(other.keywords):
            return self.kw_str() < other.kw_str()

        return len(self.keywords) < len(other.keywords)

class Keyword:
    """a Keyword includes: the text we should match, how the matching
       should be done (unique or any), which words, if present, negate
       the match, and what Slack channel this keyword is associated with"""

    def __init__(self, name, matching="any", channel=None, excludes=None):
        self.name = name
        self.matching = matching
        self.channel = channel
        self.excludes = list(set(excludes))

    def __str__(self):
        return "{}: matching={}, channel={}, NOTs={}".format(
            self.name, self.matching, self.channel, self.excludes)

class AstrophQuery:
    """ a class to define a query to the arXiv astroph papers """

    def __init__(self, start_date, end_date, max_papers, old_id=None):
        self.start_date = start_date
        self.end_date = end_date
        self.max_papers = max_papers
        self.old_id = old_id

        self.base_url = "http://export.arxiv.org/api/query?"
        self.sort_query = "max_results={}&sortBy=submittedDate&sortOrder=descending".format(
            self.max_papers)

        self.subcat = ["GA", "CO", "EP", "HE", "IM", "SR"]

    def get_cat_query(self):
        """ create the category portion of the astro ph query """

        cat_query = "%28"  # open parenthesis
        for n, s in enumerate(self.subcat):
            cat_query += "astro-ph.{}".format(s)
            if n < len(self.subcat)-1:
                cat_query += "+OR+"
            else:
                cat_query += "%29"  # close parenthesis

        return cat_query

    def get_range_query(self):
        """ get the query string for the date range """

        # here the 2000 on each date is 8:00pm
        range_str = "[{}2000+TO+{}2000]".format(self.start_date.strftime("%Y%m%d"),
                                                self.end_date.strftime("%Y%m%d"))
        range_query = "lastUpdatedDate:{}".format(range_str)
        return range_query

    def get_url(self):
        """ create the URL we will use to query arXiv """

        cat_query = self.get_cat_query()
        range_query = self.get_range_query()

        full_query = "search_query={}+AND+{}&{}".format(cat_query, range_query, self.sort_query)

        return self.base_url + full_query

    def do_query(self, keywords=None, old_id=None):
        """ perform the actual query """

        # note, in python3 this will be bytes not str
        response = urlopen(self.get_url()).read()
        response = response.replace(b"author", b"contributor")

        # this feedparser magic comes from the example of Julius Lucks / Andrea Zonca
        # https://github.com/zonca/python-parse-arxiv/blob/master/python_arXiv_parsing_example.py
        feedparser._FeedParserMixin.namespaces['http://a9.com/-/spec/opensearch/1.1/'] = 'opensearch'
        feedparser._FeedParserMixin.namespaces['http://arxiv.org/schemas/atom'] = 'arxiv'

        feed = feedparser.parse(response)

        if feed.feed.opensearch_totalresults == 0:
            sys.exit("no results found")

        results = []

        latest_id = None

        for e in feed.entries:

            arxiv_id = e.id.split("/abs/")[-1]
            title = e.title.replace("\n", " ")

            # the papers are sorted now such that the first is the
            # most recent -- we want to store this id, so the next
            # time we run the script, we can pick up from here
            if latest_id is None:
                latest_id = arxiv_id

            # now check if we hit the old_id -- this is where we
            # left off last time.  Note things may not be in id order,
            # so we keep looking through the entire list of returned
            # results.
            if old_id is not None:
                if arxiv_id < old_id:
                    continue

            # link
            for l in e.links:
                if l.rel == "alternate":
                    url = l.href

            abstract = e.summary

            # any keyword matches?
            # we do two types of matches here.  If the keyword tuple has the "any"
            # qualifier, then we don't care how it appears in the text, but if
            # it has "unique", then we want to make sure only that word matches,
            # i.e., "nova" and not "supernova".  If any of the exclude words associated
            # with the keyword are present, then we reject any match
            keys_matched = []
            channels = []
            for k in keywords:
                # first check the "NOT"s
                excluded = False
                for n in k.excludes:
                    if n in abstract.lower().replace("\n", " ") or n in title.lower():
                        # we've matched one of the excludes
                        excluded = True

                if excluded:
                    continue

                if k.matching == "any":
                    if k.name in abstract.lower().replace("\n", " ") or k.name in title.lower():
                        keys_matched.append(k.name)
                        channels.append(k.channel)

                elif k.matching == "unique":
                    qa = [l.lower().strip('\":.,!?') for l in abstract.split()]
                    qt = [l.lower().strip('\":.,!?') for l in title.split()]
                    if k.name in qa + qt:
                        keys_matched.append(k.name)
                        channels.append(k.channel)

            if keys_matched:
                results.append(Paper(arxiv_id, title, url, keys_matched, channels))

        return results, latest_id

def report(body, subject, sender, receiver):
    """ send an email """

    msg = MIMEText(body)
    msg['Subject'] = subject
    msg['From'] = sender
    msg['To'] = receiver

    try:
        sm = smtplib.SMTP('localhost')
        sm.sendmail(sender, receiver, msg.as_string())
    except smtplib.SMTPException:
        sys.exit("ERROR sending mail")

def search_astroph(keywords, old_id=None):
    """ do the actual search though astro-ph by first querying astro-ph
        for the latest papers and then looking for keyword matches"""

    today = dt.date.today()
    day = dt.timedelta(days=1)

    max_papers = 1000

    # we pick a wide-enough search range to ensure we catch papers
    # if there is a holiday

    # also, something wierd happens -- the arxiv ids appear to be
    # in descending order if you look at the "pastweek" listing
    # but the submission dates can vary wildly.  It seems that some
    # papers are held for a week or more before appearing.
    q = AstrophQuery(today - 10*day, today, max_papers, old_id=old_id)
    print(q.get_url())

    papers, last_id = q.do_query(keywords=keywords, old_id=old_id)

    papers.sort(reverse=True)

    return papers, last_id

def send_email(papers, mail=None):

    # compose the body of our e-mail
    body = ""

    # sort papers by keywords
    current_kw = None
    for p in papers:
        if not p.kw_str() == current_kw:
            current_kw = p.kw_str()
            body += "\nkeywords: {}\n\n".format(current_kw)

        body += u"{}\n".format(p)

    # e-mail it
    if not len(papers) == 0:
        if not mail is None:
            report(body, "astro-ph papers of interest",
                   "lazy-astroph@{}".format(platform.node()), mail)
        else:
            print(body)

def run(string):
    """ run a UNIX command """

    # shlex.split will preserve inner quotes
    prog = shlex.split(string)
    p0 = subprocess.Popen(prog, stdout=subprocess.PIPE,
                          stderr=subprocess.STDOUT)

    stdout0, stderr0 = p0.communicate()
    rc = p0.returncode
    p0.stdout.close()

    return stdout0, stderr0, rc

def slack_post(papers, channel_req, username=None, icon_emoji=None, webhook=None):
    """ post the information to a slack channel """

    # loop by channel
    for c in channel_req:
        channel_body = ""
        for p in papers:
            if not p.posted_to_slack:
                if c in p.channels:
                    if len(p.keywords) >= channel_req[c]:
                        keywds = ", ".join(p.keywords).strip()
                        channel_body += u"{} [{}]\n\n".format(p, keywds)
                        p.posted_to_slack = 1

        if webhook is None:
            print("channel: {}".format(c))
            print(channel_body)
            continue

        payload = {}
        payload["channel"] = c
        if username is not None:
            payload["username"] = username
        if icon_emoji is not None:
            payload["icon_emoji"] = icon_emoji
        payload["text"] = channel_body

        cmd = "curl -X POST --data-urlencode 'payload={}' {}".format(json.dumps(payload), webhook)
        run(cmd)

    sys.exit("done")

def doit():
    """ the main driver for the lazy-astroph script """

    # parse runtime parameters
    parser = argparse.ArgumentParser()

    parser.add_argument("-m", help="e-mail address to send report to",
                        type=str, default=None)
    parser.add_argument("inputs", help="inputs file containing keywords",
                        type=str, nargs=1)
    parser.add_argument("-w", help="file containing slack webhook URL",
                        type=str, default=None)
    parser.add_argument("-u", help="slack username appearing in post",
                        type=str, default=None)
    parser.add_argument("-e", help="slack icon_emoji appearing in post",
                        type=str, default=None)
    parser.add_argument("--dry_run",
                        help="don't send any mail or slack posts and don't update the marker where we left off",
                        action="store_true")
    args = parser.parse_args()

    # get the keywords
    keywords = []
    try:
        f = open(args.inputs[0], "r")
    except:
        sys.exit("ERROR: unable to open inputs file")
    else:
        channel = None
        channel_req = {}
        for line in f:
            l = line.lower().rstrip()

            if l == "":
                continue

            elif l.startswith("#") or l.startswith("@"):
                # this line defines a channel
                ch = l.split()
                channel = ch[0]
                if len(ch) == 2:
                    requires = int(ch[1].split("=")[1])
                else:
                    requires = 1
                channel_req[channel] = requires

            else:
                # this line has a keyword (and optional NOT keywords)
                if "not:" in l:
                    kw, nots = l.split("not:")
                    kw = kw.strip()
                    excludes = [x.strip() for x in nots.split(",")]
                else:
                    kw = l.strip()
                    excludes = []

                if kw[len(kw)-1] == "-":
                    matching = "unique"
                    kw = kw[:len(kw)-1]
                else:
                    matching = "any"

                keywords.append(Keyword(kw, matching=matching,
                                        channel=channel, excludes=excludes))

    # have we done this before? if so, read the .lazy_astroph file to get
    # the id of the paper we left off with
    param_file = os.path.expanduser("~") + "/.lazy_astroph"
    try:
        f = open(param_file, "r")
    except:
        old_id = None
    else:
        old_id = f.readline().rstrip()
        f.close()

    papers, last_id = search_astroph(keywords, old_id=old_id)

    if not args.dry_run:
        send_email(papers, mail=args.m)

        if not args.w is None:
            try:
                f = open(args.w)
            except:
                sys.exit("ERROR: unable to open webhook file")

            webhook = str(f.readline())
            f.close()
        else:
            webhook = None

        slack_post(papers, channel_req, icon_emoji=args.e, username=args.u, webhook=webhook)

        try:
            f = open(param_file, "w")
        except:
            sys.exit("ERROR: unable to open parameter file for writting")
        else:
            f.write(last_id)
            f.close()
    else:
        send_email(papers, mail=None)

if __name__ == "__main__":
    doit()
<|endoftext|>


<commit_before># Giles: utils.py
# Copyright 2012 Phil Bordelon
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

class Struct(object):
    # Empty class, useful for making "structs."
    pass

def booleanize(msg):
    # This returns:
    # -1 for False
    # 1 for True
    # 0 for invalid input.

    if type(msg) != str:
        return 0

    msg = msg.strip().lower()
    if msg in ('1', 'true', 't', 'yes', 'y', 'on'):
        return 1
    elif msg in ('0', 'false', 'f', 'no', 'n', 'off'):
        return -1

    return 0
<commit_msg>utils: New major utility, demangle_move().<commit_after># Giles: utils.py
# Copyright 2012 Phil Bordelon
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

class Struct(object):
    # Empty class, useful for making "structs."
    pass

def booleanize(msg):
    # This returns:
    # -1 for False
    # 1 for True
    # 0 for invalid input.

    if type(msg) != str:
        return 0

    msg = msg.strip().lower()
    if msg in ('1', 'true', 't', 'yes', 'y', 'on'):
        return 1
    elif msg in ('0', 'false', 'f', 'no', 'n', 'off'):
        return -1

    return 0

LETTERS = "abcdefghijklmnopqrstuvwxyz"
def demangle_move(move_list):

    # Most games take a move in the format Ln, where L is a letter
    # and n is some number.  They make take more than one such move,
    # for games where you move a piece from a to b.  Rather than
    # force every game to reimplement all the various ways players
    # might want to enter this, this function does the best it can
    # to return a valid set of coordinates.  It assumes zero-based
    # indexing, letter first; you may have to reverse entries, etc.
    # to get it how you want, but at least you don't have to parse
    # wacky strings.  This function will properly handle:
    # * a2
    # * a2 b2
    # * a2-b2
    # * a2,b2
    # * a2/b2
    # * a2, b2
    # ...and further coordinated versions of the same, including
    # mixed versions, ones with too many dashes, and so on.  It
    # returns a list of tuples representing the 0-indexed
    # letter/number sets, and a None if any of the entries were
    # nonsense ("aa5", "0", "b", et cetera).

    # Okay, so, to start, let's concatenate whatever list of
    # strings we got...
    move_str = " ".join(move_list)

    # ...and then replace every random character we see in the
    # string with a space.
    new_str = ""
    for char in move_str:
        if char.isalnum():
            new_str += char
        else:
            new_str += " "

    # Now we lowercase it and split it back out into components.
    new_str_bits = new_str.lower().split()

    # Now let's work on each of the bits...
    to_return = []
    for bit in new_str_bits:

        # Is it at least two and at most three characters long?
        if len(bit) < 2 or len(bit) > 3:
            return None

        # Is the first bit a letter and the rest a number?
        if not bit[0].isalpha() or not bit[1:].isdigit():
            return None

        # Okay, good.  Let's get the values.  Remember that it's zero-based,
        # so the number needs to be decremented by one.
        letter_val = LETTERS.index(bit[0])
        number_val = int(bit[1:]) - 1

        to_return.append((letter_val, number_val))

    # Since we never bailed due to bad data, let's return the list.
    return to_return
<|endoftext|>


<commit_before>"""
UML model support functions.

Functions collected in this module allow to

- create more complex UML model structures
- perform specific searches and manipulations

"""

import itertools
from gaphor.UML.uml2 import *

# '<<%s>>'
STEREOTYPE_FMT = "\xc2\xab%s\xc2\xbb"

def stereotypes_str(element, stereotypes=()):
    """
    Identify stereotypes of an UML metamodel instance and return coma
    separated stereotypes as string.

    :Parameters:
     element
        Element having stereotypes, can be None.
     stereotypes
        List of additional stereotypes, can be empty.
    """
    # generate string with stereotype names separated by coma
    if element:
        applied = (stereotype_name(st) for st in get_applied_stereotypes(element))
    else:
        applied = ()
    s = ", ".join(itertools.chain(stereotypes, applied))
    if s:
        return STEREOTYPE_FMT % s
    else:
        return ""

def stereotype_name(stereotype):
    """
    Return stereotype name suggested by UML specification. First will be
    character lowercase unless the second character is uppercase.

    :Parameters:
     stereotype
        Stereotype UML metamodel instance.
    """
    name = stereotype.name
    if not name:
        return ""
    elif len(name) > 1 and name[1].isupper():
        return name
    else:
        return name[0].lower() + name[1:]

def apply_stereotype(factory, element, stereotype):
    """
    Apply a stereotype to an element.

    :Parameters:
     factory
        UML metamodel factory.
     element
        UML metamodel class instance.
     stereotype
        UML metamodel stereotype instance.
    """
    obj = factory.create(InstanceSpecification)
    obj.classifier = stereotype
    element.appliedStereotype = obj
    return obj

def find_instances(factory, element):
    """
    Find instance specification which extend classifier `element`.
    """
    return factory.select(
        lambda e: e.isKindOf(InstanceSpecification)
        and e.classifier
        and e.classifier[0] == element
    )

def remove_stereotype(element, stereotype):
    """
    Remove a stereotype from an element.

    :Parameters:
     element
        UML metamodel element instance.
     stereotype
        UML metamodel stereotype instance.
    """
    for obj in element.appliedStereotype:
        if obj.classifier and obj.classifier[0] is stereotype:
            del element.appliedStereotype[obj]
            obj.unlink()
            break

def get_stereotypes(factory, element):
    """
    Get sorted collection of possible stereotypes for specified element.
    """
    # UML specs does not allow to extend stereotypes with stereotypes
    if isinstance(element, Stereotype):
        return ()

    cls = type(element)

    # find out names of classes, which are superclasses of element class
    names = set(c.__name__ for c in cls.__mro__ if issubclass(c, Element))

    # find stereotypes that extend element class
    classes = factory.select(lambda e: e.isKindOf(Class) and e.name in names)

    stereotypes = set(ext.ownedEnd.type for cls in classes for ext in cls.extension)
    return sorted(stereotypes, key=lambda st: st.name)

def get_applied_stereotypes(element):
    """
    Get collection of applied stereotypes to an element.
    """
    return element.appliedStereotype[:].classifier

def create_extension(factory, element, stereotype):
    """
    Extend an element with a stereotype.
    """
    ext = factory.create(Extension)
    p = factory.create(Property)
    ext_end = factory.create(ExtensionEnd)

    ext.memberEnd = p
    ext.memberEnd = ext_end
    ext.ownedEnd = ext_end
    ext_end.type = stereotype
    ext_end.aggregation = "composite"
    p.type = element
    p.name = "baseClass"
    stereotype.ownedAttribute = p

    return ext

extend_with_stereotype = create_extension

def add_slot(factory, instance, definingFeature):
    """
    Add slot to instance specification for an attribute.
    """
    slot = factory.create(Slot)
    slot.definingFeature = definingFeature
    instance.slot = slot
    return slot

def create_dependency(factory, supplier, client):
    dep = factory.create(Dependency)
    dep.supplier = supplier
    dep.client = client
    return dep

def create_realization(factory, realizingClassifier, abstraction):
    dep = factory.create(Realization)
    dep.realizingClassifier = realizingClassifier
    dep.abstraction = abstraction
    return dep

def create_generalization(factory, general, specific):
    gen = factory.create(Generalization)
    gen.general = general
    gen.specific = specific
    return gen

def create_implementation(factory, contract, implementatingClassifier):
    impl = factory.create(Implementation)
    impl.contract = contract
    impl.implementatingClassifier = implementatingClassifier
    return impl

def create_association(factory, type_a, type_b):
    """
    Create an association between two items.
    """
    assoc = factory.create(Association)
    end_a = factory.create(Property)
    end_b = factory.create(Property)
    assoc.memberEnd = end_a
    assoc.memberEnd = end_b
    end_a.type = type_a
    end_b.type = type_b
    # set default navigability (unknown)
    set_navigability(assoc, end_a, None)
    set_navigability(assoc, end_b, None)
    return assoc

def set_navigability(assoc, end, nav):
    """
    Set navigability of an association end (property).

    There are free possible values for ``nav`` parameter

     True
        association end is navigable
     False
        association end is not navigable
     None
        association end navigability is unkown

    There are two ways of specifing than an end is navigable

    - an end is in Association.navigableOwnedEnd collection
    - an end is class (interface) attribute (stored in Class.ownedAttribute
      collection)

    Let's consider the graph::

        A -----> B
          y    x

    There two association ends A.x and B.y, A.x is navigable.

    Therefore navigable association ends are constructed in following way

    - if A is a class or an interface, then A.x is an attribute owned by A
    - if A is other classifier, then association is more general
      relationship; it may mean that participating instance of B can be
      "accessed efficiently"
      - i.e. when A is a Component, then association may be some compositing
        relationship
      - when A and B are instances of Node class, then it is a
        communication path

    Therefore navigable association end may be stored as one of
    - {Class,Interface}.ownedAttribute due to their capabilities of
      editing owned members
    - Association.navigableOwnedEnd

    When an end has unknown (unspecified) navigability, then it is owned by
    association (but not by classifier).

    When an end is non-navigable, then it is just member of an association.
    """
    # remove "navigable" and "unspecified" navigation indicators first
    if type(end.type) in (Class, Interface):
        owner = end.opposite.type
        if end in owner.ownedAttribute:
            owner.ownedAttribute.remove(end)
    if end in assoc.ownedEnd:
        assoc.ownedEnd.remove(end)
    if end in assoc.navigableOwnedEnd:
        assoc.navigableOwnedEnd.remove(end)
    assert end not in assoc.ownedEnd
    assert end not in assoc.navigableOwnedEnd

    if nav is True:
        if type(end.type) in (Class, Interface):
            owner = end.opposite.type
            owner.ownedAttribute = end
        else:
            assoc.navigableOwnedEnd = end
    elif nav is None:
        assoc.ownedEnd = end
    # elif nav is False, non-navigable

def dependency_type(client, supplier):
    """
    Determine dependency type between client (tail) and supplier
    (arrowhead).

    There can be different dependencies detected automatically

    - usage when supplier is an interface
    - realization when client is component and supplier is a classifier

    If none of above is detected then standard dependency is determined.
    """
    dt = Dependency

    # test interface first as it is a classifier
    if isinstance(supplier, Interface):
        dt = Usage
    elif isinstance(client, Component) and isinstance(supplier, Classifier):
        dt = Realization

    return dt

def create_message(factory, msg, inverted=False):
    """
    Create new message based on speciied message.

    If inverted is set to True, then inverted message is created.
    """
    message = factory.create(Message)
    send = None
    receive = None

    if msg.sendEvent:
        send = factory.create(MessageOccurrenceSpecification)
        sl = msg.sendEvent.covered
        send.covered = sl
    if msg.receiveEvent:
        receive = factory.create(MessageOccurrenceSpecification)
        rl = msg.receiveEvent.covered
        receive.covered = rl

    if inverted:
        # inverted message goes in different direction, than original
        # message
        message.sendEvent = receive
        message.receiveEvent = send
    else:
        message.sendEvent = send
        message.receiveEvent = receive
    return message

# vim:sw=4:et:ai
<commit_msg>"fix" how stereotype is displayed<commit_after>"""
UML model support functions.

Functions collected in this module allow to

- create more complex UML model structures
- perform specific searches and manipulations

"""

import itertools
from gaphor.UML.uml2 import *

# '<<%s>>'
STEREOTYPE_FMT = "<<%s>>"

def stereotypes_str(element, stereotypes=()):
    """
    Identify stereotypes of an UML metamodel instance and return coma
    separated stereotypes as string.

    :Parameters:
     element
        Element having stereotypes, can be None.
     stereotypes
        List of additional stereotypes, can be empty.
    """
    # generate string with stereotype names separated by coma
    if element:
        applied = (stereotype_name(st) for st in get_applied_stereotypes(element))
    else:
        applied = ()
    s = ", ".join(itertools.chain(stereotypes, applied))
    if s:
        return STEREOTYPE_FMT % s
    else:
        return ""

def stereotype_name(stereotype):
    """
    Return stereotype name suggested by UML specification. First will be
    character lowercase unless the second character is uppercase.

    :Parameters:
     stereotype
        Stereotype UML metamodel instance.
    """
    name = stereotype.name
    if not name:
        return ""
    elif len(name) > 1 and name[1].isupper():
        return name
    else:
        return name[0].lower() + name[1:]

def apply_stereotype(factory, element, stereotype):
    """
    Apply a stereotype to an element.

    :Parameters:
     factory
        UML metamodel factory.
     element
        UML metamodel class instance.
     stereotype
        UML metamodel stereotype instance.
    """
    obj = factory.create(InstanceSpecification)
    obj.classifier = stereotype
    element.appliedStereotype = obj
    return obj

def find_instances(factory, element):
    """
    Find instance specification which extend classifier `element`.
    """
    return factory.select(
        lambda e: e.isKindOf(InstanceSpecification)
        and e.classifier
        and e.classifier[0] == element
    )

def remove_stereotype(element, stereotype):
    """
    Remove a stereotype from an element.

    :Parameters:
     element
        UML metamodel element instance.
     stereotype
        UML metamodel stereotype instance.
    """
    for obj in element.appliedStereotype:
        if obj.classifier and obj.classifier[0] is stereotype:
            del element.appliedStereotype[obj]
            obj.unlink()
            break

def get_stereotypes(factory, element):
    """
    Get sorted collection of possible stereotypes for specified element.
    """
    # UML specs does not allow to extend stereotypes with stereotypes
    if isinstance(element, Stereotype):
        return ()

    cls = type(element)

    # find out names of classes, which are superclasses of element class
    names = set(c.__name__ for c in cls.__mro__ if issubclass(c, Element))

    # find stereotypes that extend element class
    classes = factory.select(lambda e: e.isKindOf(Class) and e.name in names)

    stereotypes = set(ext.ownedEnd.type for cls in classes for ext in cls.extension)
    return sorted(stereotypes, key=lambda st: st.name)

def get_applied_stereotypes(element):
    """
    Get collection of applied stereotypes to an element.
    """
    return element.appliedStereotype[:].classifier

def create_extension(factory, element, stereotype):
    """
    Extend an element with a stereotype.
    """
    ext = factory.create(Extension)
    p = factory.create(Property)
    ext_end = factory.create(ExtensionEnd)

    ext.memberEnd = p
    ext.memberEnd = ext_end
    ext.ownedEnd = ext_end
    ext_end.type = stereotype
    ext_end.aggregation = "composite"
    p.type = element
    p.name = "baseClass"
    stereotype.ownedAttribute = p

    return ext

extend_with_stereotype = create_extension

def add_slot(factory, instance, definingFeature):
    """
    Add slot to instance specification for an attribute.
    """
    slot = factory.create(Slot)
    slot.definingFeature = definingFeature
    instance.slot = slot
    return slot

def create_dependency(factory, supplier, client):
    dep = factory.create(Dependency)
    dep.supplier = supplier
    dep.client = client
    return dep

def create_realization(factory, realizingClassifier, abstraction):
    dep = factory.create(Realization)
    dep.realizingClassifier = realizingClassifier
    dep.abstraction = abstraction
    return dep

def create_generalization(factory, general, specific):
    gen = factory.create(Generalization)
    gen.general = general
    gen.specific = specific
    return gen

def create_implementation(factory, contract, implementatingClassifier):
    impl = factory.create(Implementation)
    impl.contract = contract
    impl.implementatingClassifier = implementatingClassifier
    return impl

def create_association(factory, type_a, type_b):
    """
    Create an association between two items.
    """
    assoc = factory.create(Association)
    end_a = factory.create(Property)
    end_b = factory.create(Property)
    assoc.memberEnd = end_a
    assoc.memberEnd = end_b
    end_a.type = type_a
    end_b.type = type_b
    # set default navigability (unknown)
    set_navigability(assoc, end_a, None)
    set_navigability(assoc, end_b, None)
    return assoc

def set_navigability(assoc, end, nav):
    """
    Set navigability of an association end (property).

    There are free possible values for ``nav`` parameter

     True
        association end is navigable
     False
        association end is not navigable
     None
        association end navigability is unkown

    There are two ways of specifing than an end is navigable

    - an end is in Association.navigableOwnedEnd collection
    - an end is class (interface) attribute (stored in Class.ownedAttribute
      collection)

    Let's consider the graph::

        A -----> B
          y    x

    There two association ends A.x and B.y, A.x is navigable.

    Therefore navigable association ends are constructed in following way

    - if A is a class or an interface, then A.x is an attribute owned by A
    - if A is other classifier, then association is more general
      relationship; it may mean that participating instance of B can be
      "accessed efficiently"
      - i.e. when A is a Component, then association may be some compositing
        relationship
      - when A and B are instances of Node class, then it is a
        communication path

    Therefore navigable association end may be stored as one of
    - {Class,Interface}.ownedAttribute due to their capabilities of
      editing owned members
    - Association.navigableOwnedEnd

    When an end has unknown (unspecified) navigability, then it is owned by
    association (but not by classifier).

    When an end is non-navigable, then it is just member of an association.
    """
    # remove "navigable" and "unspecified" navigation indicators first
    if type(end.type) in (Class, Interface):
        owner = end.opposite.type
        if end in owner.ownedAttribute:
            owner.ownedAttribute.remove(end)
    if end in assoc.ownedEnd:
        assoc.ownedEnd.remove(end)
    if end in assoc.navigableOwnedEnd:
        assoc.navigableOwnedEnd.remove(end)
    assert end not in assoc.ownedEnd
    assert end not in assoc.navigableOwnedEnd

    if nav is True:
        if type(end.type) in (Class, Interface):
            owner = end.opposite.type
            owner.ownedAttribute = end
        else:
            assoc.navigableOwnedEnd = end
    elif nav is None:
        assoc.ownedEnd = end
    # elif nav is False, non-navigable

def dependency_type(client, supplier):
    """
    Determine dependency type between client (tail) and supplier
    (arrowhead).

    There can be different dependencies detected automatically

    - usage when supplier is an interface
    - realization when client is component and supplier is a classifier

    If none of above is detected then standard dependency is determined.
    """
    dt = Dependency

    # test interface first as it is a classifier
    if isinstance(supplier, Interface):
        dt = Usage
    elif isinstance(client, Component) and isinstance(supplier, Classifier):
        dt = Realization

    return dt

def create_message(factory, msg, inverted=False):
    """
    Create new message based on speciied message.

    If inverted is set to True, then inverted message is created.
    """
    message = factory.create(Message)
    send = None
    receive = None

    if msg.sendEvent:
        send = factory.create(MessageOccurrenceSpecification)
        sl = msg.sendEvent.covered
        send.covered = sl
    if msg.receiveEvent:
        receive = factory.create(MessageOccurrenceSpecification)
        rl = msg.receiveEvent.covered
        receive.covered = rl

    if inverted:
        # inverted message goes in different direction, than original
        # message
        message.sendEvent = receive
        message.receiveEvent = send
    else:
        message.sendEvent = send
        message.receiveEvent = receive
    return message

# vim:sw=4:et:ai
<|endoftext|>


<commit_before>from os import system, path, remove, chdir, getcwd, stat
import re
import platform

# System dependent functions

# Note, sometime in 2018 ImageMagick's convert program by default
# disallowed pdf file conversions.  A work-around is to edit
# /etc/ImageMagick-6/policy.xml to enable this conversion.

def tmpfilename(suffix=''):

    from tempfile import gettempdir, NamedTemporaryFile

    # Searches using TMPDIR, TEMP, TMP environment variables
    tempdir = gettempdir()

    filename = NamedTemporaryFile(suffix=suffix, dir=tempdir,
                                  delete=False).name
    return filename

def convert_pdf_svg(pdf_filename, svg_filename):

    system('pdf2svg %s %s' % (pdf_filename, svg_filename))
    if not path.exists(svg_filename):
        raise RuntimeError('Could not generate %s with pdf2svg.  Is it installed?' %
                           svg_filename)

def convert_pdf_png_convert(pdf_filename, png_filename, dpi=300):

    program = 'convert'
    if platform.system() == 'Windows':
        program = 'magick convert'

    command = '%s -density %d %s %s' % (program, int(dpi), pdf_filename,
                                        png_filename)
    system(command)

    if not path.exists(png_filename):
        raise RuntimeError('Could not generate %s with convert' %
                           png_filename)
    if stat(png_filename).st_size == 0:
        raise RuntimeError('Could not generate %s with convert, empty file' %
                           png_filename)

def convert_pdf_png_ghostscript(pdf_filename, png_filename, dpi=300):

    program = 'gs'
    if platform.system() == 'Windows':
        program = 'gswin32'
        if platform.machine().endswith('64'):
            program = 'gswin64'

    command = '%s -q -dQUIET -dSAFER -dBATCH -dNOPAUSE -dNOPROMPT -dMaxBitmap=500000000 -dAlignToPixels=0 -dGridFitTT=2 -sDEVICE=pngalpha -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -r%dx%d -sOutputFile=%s %s' % (program, int(dpi), int(dpi), png_filename, pdf_filename)

    system(command)

def convert_pdf_png_pdftoppm(pdf_filename, png_filename, dpi=300):

    system('pdftoppm -r %d  -png %s -thinlinemode shape > %s' %
           (int(dpi), pdf_filename, png_filename))

    if not path.exists(png_filename):
        raise RuntimeError('Could not generate %s with pdftoppm' %
                           png_filename)

def convert_pdf_png(pdf_filename, png_filename, dpi=300):

    try:
        convert_pdf_png_ghostscript(pdf_filename, png_filename, dpi)
    except:
        try:
            convert_pdf_png_convert(pdf_filename, png_filename, dpi)
        except:
            convert_pdf_png_pdftoppm(pdf_filename, png_filename, dpi)

def latex_cleanup(tex_filename, wanted_filename=''):

    root, ext = path.splitext(tex_filename)
    exts = ['.tex', '.aux', '.log', '.pdf']
    filenames = [root + ext for ext in exts]

    if wanted_filename in filenames:
        filenames.remove(wanted_filename)

    for filename in filenames:
        if path.exists(filename):
            remove(filename)

def run_latex(tex_filename):

    root, ext = path.splitext(tex_filename)
    dirname = path.dirname(tex_filename)
    baseroot = path.basename(root)
    cwd = getcwd()
    if dirname != '':
        chdir(path.abspath(dirname))

    system('pdflatex -interaction batchmode %s.tex' % baseroot)

    if dirname != '':
        chdir(cwd)

    return root + '.pdf'

def run_dot(dotfilename, filename):

    base, ext = path.splitext(filename)
    system('dot -T %s -o %s %s' % (ext[1:], filename, dotfilename))
    remove(dotfilename)
    return

def circuitikz_version():

    content = r"""
    \documentclass[a4paper]{standalone}
    \usepackage{circuitikz}
    \begin{document}
    \end{document}"""

    tex_filename = tmpfilename('.tex')
    open(tex_filename, 'w').write(content)

    log_filename = tex_filename.replace('.tex', '.log')
    run_latex(tex_filename)

    with open(log_filename, 'rt') as logfile:
        lines = logfile.readlines()
        for m, line in enumerate(lines):
            if line.startswith('Package: circuitikz'):
                match = re.search(r'circuitikz ([0-9/]+)', line)
                date = match.group(1)
                version = lines[m + 1].strip()
                return date, version
    return None, None

<commit_msg>Use call to suppress pdflatex output<commit_after>from os import system, path, remove, chdir, getcwd, stat
from subprocess import call, DEVNULL
import re
import platform

# System dependent functions

# Note, sometime in 2018 ImageMagick's convert program by default
# disallowed pdf file conversions.  A work-around is to edit
# /etc/ImageMagick-6/policy.xml to enable this conversion.

def tmpfilename(suffix=''):

    from tempfile import gettempdir, NamedTemporaryFile

    # Searches using TMPDIR, TEMP, TMP environment variables
    tempdir = gettempdir()

    filename = NamedTemporaryFile(suffix=suffix, dir=tempdir,
                                  delete=False).name
    return filename

def convert_pdf_svg(pdf_filename, svg_filename):

    system('pdf2svg %s %s' % (pdf_filename, svg_filename))
    if not path.exists(svg_filename):
        raise RuntimeError('Could not generate %s with pdf2svg.  Is it installed?' %
                           svg_filename)

def convert_pdf_png_convert(pdf_filename, png_filename, dpi=300):

    program = 'convert'
    if platform.system() == 'Windows':
        program = 'magick convert'

    command = '%s -density %d %s %s' % (program, int(dpi), pdf_filename,
                                        png_filename)
    system(command)

    if not path.exists(png_filename):
        raise RuntimeError('Could not generate %s with convert' %
                           png_filename)
    if stat(png_filename).st_size == 0:
        raise RuntimeError('Could not generate %s with convert, empty file' %
                           png_filename)

def convert_pdf_png_ghostscript(pdf_filename, png_filename, dpi=300):

    program = 'gs'
    if platform.system() == 'Windows':
        program = 'gswin32'
        if platform.machine().endswith('64'):
            program = 'gswin64'

    command = '%s -q -dQUIET -dSAFER -dBATCH -dNOPAUSE -dNOPROMPT -dMaxBitmap=500000000 -dAlignToPixels=0 -dGridFitTT=2 -sDEVICE=pngalpha -dTextAlphaBits=4 -dGraphicsAlphaBits=4 -r%dx%d -sOutputFile=%s %s' % (program, int(dpi), int(dpi), png_filename, pdf_filename)

    system(command)

def convert_pdf_png_pdftoppm(pdf_filename, png_filename, dpi=300):

    system('pdftoppm -r %d  -png %s -thinlinemode shape > %s' %
           (int(dpi), pdf_filename, png_filename))

    if not path.exists(png_filename):
        raise RuntimeError('Could not generate %s with pdftoppm' %
                           png_filename)

def convert_pdf_png(pdf_filename, png_filename, dpi=300):

    try:
        convert_pdf_png_ghostscript(pdf_filename, png_filename, dpi)
    except:
        try:
            convert_pdf_png_convert(pdf_filename, png_filename, dpi)
        except:
            convert_pdf_png_pdftoppm(pdf_filename, png_filename, dpi)

def latex_cleanup(tex_filename, wanted_filename=''):

    root, ext = path.splitext(tex_filename)
    exts = ['.tex', '.aux', '.log', '.pdf']
    filenames = [root + ext for ext in exts]

    if wanted_filename in filenames:
        filenames.remove(wanted_filename)

    for filename in filenames:
        if path.exists(filename):
            remove(filename)

def run_latex(tex_filename):

    root, ext = path.splitext(tex_filename)
    dirname = path.dirname(tex_filename)
    baseroot = path.basename(root)
    cwd = getcwd()
    if dirname != '':
        chdir(path.abspath(dirname))

    call(['pdflatex', '-interaction', 'batchmode', '%s.tex' %
          baseroot], stderr=DEVNULL, stdout=DEVNULL)

    if dirname != '':
        chdir(cwd)

    return root + '.pdf'

def run_dot(dotfilename, filename):

    base, ext = path.splitext(filename)
    system('dot -T %s -o %s %s' % (ext[1:], filename, dotfilename))
    remove(dotfilename)
    return

def circuitikz_version():

    content = r"""
    \documentclass[a4paper]{standalone}
    \usepackage{circuitikz}
    \begin{document}
    \end{document}"""

    tex_filename = tmpfilename('.tex')
    open(tex_filename, 'w').write(content)

    log_filename = tex_filename.replace('.tex', '.log')
    run_latex(tex_filename)

    with open(log_filename, 'rt') as logfile:
        lines = logfile.readlines()
        for m, line in enumerate(lines):
            if line.startswith('Package: circuitikz'):
                match = re.search(r'circuitikz ([0-9/]+)', line)
                date = match.group(1)
                version = lines[m + 1].strip()
                return date, version
    return None, None

<|endoftext|>


<commit_before># coding: utf-8

""" Utilities for nonlinear dynamics """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import logging

# Third-party
import numpy as np

# Project
from ..util import gram_schmidt

__all__ = ['lyapunov_spectrum', 'fast_lyapunov_max', 'lyapunov_max']

# Create logger
logger = logging.getLogger(__name__)

def lyapunov_spectrum(w0, integrator, dt, nsteps, t1=0., deviation_vecs=None):
    """ Compute the spectrum of Lyapunov exponents given equations of motions
        for small deviations.

        Parameters
        ----------
        w0 : array_like
            Initial conditions for all phase-space coordinates.
        integrator : gary.Integrator
            An instantiated Integrator object. Must have a run() method.
        dt : numeric
            Timestep.
        nsteps : int
            Number of steps to run for.
        t1 : numeric (optional)
            Time of initial conditions. Assumed to be t=0.
        deviation_vecs : array_like (optional)
            Specify your own initial deviation vectors.

    """

    w0 = np.atleast_2d(w0)

    # phase-space dimensionality
    if w0.shape[0] > 1:
        raise ValueError("Initial condition vector ")
    ndim_ps = w0.shape[1]

    if deviation_vecs is None:
        # initialize (ndim_ps) deviation vectors
        A = np.zeros((ndim_ps,ndim_ps))
        for ii in range(ndim_ps):
            A[ii] = np.random.normal(0.,1.,size=ndim_ps)
            A[ii] /= np.linalg.norm(A[ii])

    else:
        raise NotImplementedError()

    all_w0 = np.zeros((ndim_ps,ndim_ps*2))
    for ii in range(ndim_ps):
        all_w0[ii] = np.append(w0,A[ii])

    # array to store the full, main orbit
    full_w = np.zeros((nsteps+1,ndim_ps))
    full_w[0] = w0
    full_ts = np.zeros((nsteps+1,))
    full_ts[0] = t1

    # arrays to store the Lyapunov exponents and times
    lyap = np.zeros((nsteps+1,ndim_ps))
    rhi = np.zeros((nsteps+1,ndim_ps))  # sum of logs

    ts = np.zeros(nsteps+1)
    time = t1
    for ii in range(1,nsteps+1):
        tt,ww = integrator.run(all_w0, dt=dt, nsteps=1, t1=time)
        time += dt

        alf = gram_schmidt(ww[-1,:,ndim_ps:])
        rhi[ii] = rhi[ii-1] + np.log(alf)
        lyap[ii] = rhi[ii]/time

        ts[ii] = time
        full_w[ii:ii+1] = ww[1:,0,:ndim_ps]
        full_ts[ii:ii+1] = tt[1:]
        all_w0 = ww[-1].copy()

    return lyap, full_ts, full_w

def fast_lyapunov_max(w0, potential, dt, nsteps, d0=1e-5,
                      nsteps_per_pullback=10, noffset_orbits=8, t1=0.):
    from ..integrate.dopri.wrap_dop853 import dop853_lyapunov

    if not hasattr(potential, 'c_instance'):
        raise TypeError("Input potential must be a CPotential subclass.")

    t,w,l = dop853_lyapunov(potential.c_instance, w0,
                            dt, nsteps, t1, 1E-8, 1E-8,
                            d0, nsteps_per_pullback, noffset_orbits)

    return l,t,w

def lyapunov_max(w0, integrator, dt, nsteps, d0=1e-5, nsteps_per_pullback=10,
                 noffset=8, t1=0.):
    """
    Compute the maximum Lyapunov exponent of an orbit by integrating many
    nearby orbits (``noffset``) separated with isotropically distributed
    directions but the same initial deviation length, ``d0``. This algorithm
    re-normalizes the offset orbits every ``nsteps_per_pullback`` steps.

    Parameters
    ----------
    w0 : array_like
        Initial conditions for all phase-space coordinates.
    integrator : gary.Integrator
        An instantiated Integrator object. Must have a run() method.
    dt : numeric
        Timestep.
    nsteps : int
        Number of steps to run for.
    d0 : numeric (optional)
        The initial separation.
    nsteps_per_pullback : int (optional)
        Number of steps to run before re-normalizing the offset vectors.
    noffset : int (optional)
        Number of offset orbits to run.
    t1 : numeric (optional)
        Time of initial conditions. Assumed to be t=0.

    Returns
    -------
    LEs : :class:`numpy.ndarray`
        Lyapunov exponents calculated from each offset / deviation orbit.
    ts : :class:`numpy.ndarray`
        Array of times from integrating main orbit.
    ws : :class:`numpy.ndarray`
        All orbits -- main / parent orbit is index 0, all others are the
        full orbits of the deviations. TODO: right now, only returns parent
        orbit.
    """

    w0 = np.atleast_2d(w0)

    # number of iterations
    niter = nsteps // nsteps_per_pullback
    ndim = w0.shape[1]

    # define offset vectors to start the offset orbits on
    d0_vec = np.random.uniform(size=(noffset,ndim))
    d0_vec /= np.linalg.norm(d0_vec, axis=1)[:,np.newaxis]
    d0_vec *= d0

    w_offset = w0 + d0_vec
    all_w0 = np.vstack((w0,w_offset))

    # array to store the full, main orbit
    full_w = np.zeros((nsteps+1,ndim))
    full_w[0] = w0
    full_ts = np.zeros((nsteps+1,))
    full_ts[0] = t1

    # arrays to store the Lyapunov exponents and times
    LEs = np.zeros((niter,noffset))
    ts = np.zeros_like(LEs)
    time = t1
    for i in range(1,niter+1):
        ii = i * nsteps_per_pullback

        tt,ww = integrator.run(all_w0, dt=dt, nsteps=nsteps_per_pullback, t1=time)
        time += dt*nsteps_per_pullback

        main_w = ww[-1,0][np.newaxis]
        d1 = ww[-1,1:] - main_w
        d1_mag = np.linalg.norm(d1, axis=1)

        LEs[i-1] = np.log(d1_mag/d0)
        ts[i-1] = time

        w_offset = ww[-1,0] + d0 * d1 / d1_mag[:,np.newaxis]
        all_w0 = np.vstack((ww[-1,0],w_offset))

        full_w[(i-1)*nsteps_per_pullback+1:ii+1] = ww[1:,0]
        full_ts[(i-1)*nsteps_per_pullback+1:ii+1] = tt[1:]

    LEs = np.array([LEs[:ii].sum(axis=0)/ts[ii-1] for ii in range(1,niter)])

    return LEs, full_ts, full_w

# def sali(w0, integrator, dt, nsteps, t1=0., deviation_vecs=None):
#     """ Compute the Smaller Alignment Index (SALI)
#         See: Skokos, Ch. 2001, J. Phys. A: Math. Gen., 34, 10029-10043

#         Parameters
#         ----------
#         w0 : array_like
#             Initial conditions for all phase-space coordinates.
#         integrator : gary.Integrator
#             An instantiated Integrator object. Must have a run() method.
#         dt : numeric
#             Timestep.
#         nsteps : int
#             Number of steps to run for.
#         d0 : numeric (optional)
#             The initial separation.
#         nsteps_per_pullback : int (optional)
#             Number of steps to run before re-normalizing the offset vectors.
#         noffset : int (optional)
#             Number of offset orbits to run.
#         t1 : numeric (optional)
#             Time of initial conditions. Assumed to be t=0.

#     """

#     w0 = np.atleast_2d(w0)

#     # phase-space dimensionality
#     if w0.shape[0] > 1:
#         raise ValueError("Initial condition vector ")
#     ndim_ps = w0.shape[1]

#     if deviation_vecs is None:
#         # initialize (ndim_ps) deviation vectors
#         A = np.zeros((ndim_ps,ndim_ps))
#         for ii in range(ndim_ps):
#             A[ii] = np.random.normal(0.,1.,size=ndim_ps)
#             A[ii] /= np.linalg.norm(A[ii])

#         vec = gram_schmidt(A)
#         A = A[:2]

#     else:
#         raise NotImplementedError()

#     all_w0 = np.zeros((2,ndim_ps*2))
#     for ii in range(2):
#         all_w0[ii] = np.append(w0,A[ii])

#     # array to store the full, main orbit
#     full_w = np.zeros((nsteps+1,ndim_ps))
#     full_w[0] = w0
#     full_ts = np.zeros((nsteps+1,))
#     full_ts[0] = t1

#     # arrays to store the sali
#     sali = np.zeros((nsteps+1,))

#     time = t1
#     for ii in range(1,nsteps+1):
#         tt,ww = integrator.run(all_w0, dt=dt, nsteps=1, t1=time)
#         time += dt

#         dm = np.sqrt(np.sum((ww[-1,0,ndim_ps:] - ww[-1,1,ndim_ps:])**2))
#         dq = np.sqrt(np.sum((ww[-1,0,ndim_ps:] + ww[-1,1,ndim_ps:])**2))
#         sali[ii] = min(dm, dq)

#         # renormalize
#         ww[-1,0,ndim_ps:] = ww[-1,0,ndim_ps:] / np.linalg.norm(ww[-1,0,ndim_ps:])
#         ww[-1,1,ndim_ps:] = ww[-1,1,ndim_ps:] / np.linalg.norm(ww[-1,1,ndim_ps:])

#         full_w[ii:ii+1] = ww[-1,0,:ndim_ps]
#         full_ts[ii:ii+1] = time
#         all_w0 = ww[-1].copy()

#     return sali, full_ts, full_w
<commit_msg>added atol rtol<commit_after># coding: utf-8

""" Utilities for nonlinear dynamics """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import logging

# Third-party
import numpy as np

# Project
from ..util import gram_schmidt

__all__ = ['lyapunov_spectrum', 'fast_lyapunov_max', 'lyapunov_max']

# Create logger
logger = logging.getLogger(__name__)

def lyapunov_spectrum(w0, integrator, dt, nsteps, t1=0., deviation_vecs=None):
    """ Compute the spectrum of Lyapunov exponents given equations of motions
        for small deviations.

        Parameters
        ----------
        w0 : array_like
            Initial conditions for all phase-space coordinates.
        integrator : gary.Integrator
            An instantiated Integrator object. Must have a run() method.
        dt : numeric
            Timestep.
        nsteps : int
            Number of steps to run for.
        t1 : numeric (optional)
            Time of initial conditions. Assumed to be t=0.
        deviation_vecs : array_like (optional)
            Specify your own initial deviation vectors.

    """

    w0 = np.atleast_2d(w0)

    # phase-space dimensionality
    if w0.shape[0] > 1:
        raise ValueError("Initial condition vector ")
    ndim_ps = w0.shape[1]

    if deviation_vecs is None:
        # initialize (ndim_ps) deviation vectors
        A = np.zeros((ndim_ps,ndim_ps))
        for ii in range(ndim_ps):
            A[ii] = np.random.normal(0.,1.,size=ndim_ps)
            A[ii] /= np.linalg.norm(A[ii])

    else:
        raise NotImplementedError()

    all_w0 = np.zeros((ndim_ps,ndim_ps*2))
    for ii in range(ndim_ps):
        all_w0[ii] = np.append(w0,A[ii])

    # array to store the full, main orbit
    full_w = np.zeros((nsteps+1,ndim_ps))
    full_w[0] = w0
    full_ts = np.zeros((nsteps+1,))
    full_ts[0] = t1

    # arrays to store the Lyapunov exponents and times
    lyap = np.zeros((nsteps+1,ndim_ps))
    rhi = np.zeros((nsteps+1,ndim_ps))  # sum of logs

    ts = np.zeros(nsteps+1)
    time = t1
    for ii in range(1,nsteps+1):
        tt,ww = integrator.run(all_w0, dt=dt, nsteps=1, t1=time)
        time += dt

        alf = gram_schmidt(ww[-1,:,ndim_ps:])
        rhi[ii] = rhi[ii-1] + np.log(alf)
        lyap[ii] = rhi[ii]/time

        ts[ii] = time
        full_w[ii:ii+1] = ww[1:,0,:ndim_ps]
        full_ts[ii:ii+1] = tt[1:]
        all_w0 = ww[-1].copy()

    return lyap, full_ts, full_w

def fast_lyapunov_max(w0, potential, dt, nsteps, d0=1e-5,
                      nsteps_per_pullback=10, noffset_orbits=2, t1=0.,
                      atol=1E-9, rtol=1E-8):
    from ..integrate._dop853 import dop853_lyapunov

    if not hasattr(potential, 'c_instance'):
        raise TypeError("Input potential must be a CPotential subclass.")

    t,w,l = dop853_lyapunov(potential.c_instance, w0,
                            dt, nsteps, t1, atol, rtol,
                            d0, nsteps_per_pullback, noffset_orbits)

    return l,t,w

def lyapunov_max(w0, integrator, dt, nsteps, d0=1e-5, nsteps_per_pullback=10,
                 noffset=8, t1=0.):
    """
    Compute the maximum Lyapunov exponent of an orbit by integrating many
    nearby orbits (``noffset``) separated with isotropically distributed
    directions but the same initial deviation length, ``d0``. This algorithm
    re-normalizes the offset orbits every ``nsteps_per_pullback`` steps.

    Parameters
    ----------
    w0 : array_like
        Initial conditions for all phase-space coordinates.
    integrator : gary.Integrator
        An instantiated Integrator object. Must have a run() method.
    dt : numeric
        Timestep.
    nsteps : int
        Number of steps to run for.
    d0 : numeric (optional)
        The initial separation.
    nsteps_per_pullback : int (optional)
        Number of steps to run before re-normalizing the offset vectors.
    noffset : int (optional)
        Number of offset orbits to run.
    t1 : numeric (optional)
        Time of initial conditions. Assumed to be t=0.

    Returns
    -------
    LEs : :class:`numpy.ndarray`
        Lyapunov exponents calculated from each offset / deviation orbit.
    ts : :class:`numpy.ndarray`
        Array of times from integrating main orbit.
    ws : :class:`numpy.ndarray`
        All orbits -- main / parent orbit is index 0, all others are the
        full orbits of the deviations. TODO: right now, only returns parent
        orbit.
    """

    w0 = np.atleast_2d(w0)

    # number of iterations
    niter = nsteps // nsteps_per_pullback
    ndim = w0.shape[1]

    # define offset vectors to start the offset orbits on
    d0_vec = np.random.uniform(size=(noffset,ndim))
    d0_vec /= np.linalg.norm(d0_vec, axis=1)[:,np.newaxis]
    d0_vec *= d0

    w_offset = w0 + d0_vec
    all_w0 = np.vstack((w0,w_offset))

    # array to store the full, main orbit
    full_w = np.zeros((nsteps+1,ndim))
    full_w[0] = w0
    full_ts = np.zeros((nsteps+1,))
    full_ts[0] = t1

    # arrays to store the Lyapunov exponents and times
    LEs = np.zeros((niter,noffset))
    ts = np.zeros_like(LEs)
    time = t1
    for i in range(1,niter+1):
        ii = i * nsteps_per_pullback

        tt,ww = integrator.run(all_w0, dt=dt, nsteps=nsteps_per_pullback, t1=time)
        time += dt*nsteps_per_pullback

        main_w = ww[-1,0][np.newaxis]
        d1 = ww[-1,1:] - main_w
        d1_mag = np.linalg.norm(d1, axis=1)

        LEs[i-1] = np.log(d1_mag/d0)
        ts[i-1] = time

        w_offset = ww[-1,0] + d0 * d1 / d1_mag[:,np.newaxis]
        all_w0 = np.vstack((ww[-1,0],w_offset))

        full_w[(i-1)*nsteps_per_pullback+1:ii+1] = ww[1:,0]
        full_ts[(i-1)*nsteps_per_pullback+1:ii+1] = tt[1:]

    LEs = np.array([LEs[:ii].sum(axis=0)/ts[ii-1] for ii in range(1,niter)])

    return LEs, full_ts, full_w

# def sali(w0, integrator, dt, nsteps, t1=0., deviation_vecs=None):
#     """ Compute the Smaller Alignment Index (SALI)
#         See: Skokos, Ch. 2001, J. Phys. A: Math. Gen., 34, 10029-10043

#         Parameters
#         ----------
#         w0 : array_like
#             Initial conditions for all phase-space coordinates.
#         integrator : gary.Integrator
#             An instantiated Integrator object. Must have a run() method.
#         dt : numeric
#             Timestep.
#         nsteps : int
#             Number of steps to run for.
#         d0 : numeric (optional)
#             The initial separation.
#         nsteps_per_pullback : int (optional)
#             Number of steps to run before re-normalizing the offset vectors.
#         noffset : int (optional)
#             Number of offset orbits to run.
#         t1 : numeric (optional)
#             Time of initial conditions. Assumed to be t=0.

#     """

#     w0 = np.atleast_2d(w0)

#     # phase-space dimensionality
#     if w0.shape[0] > 1:
#         raise ValueError("Initial condition vector ")
#     ndim_ps = w0.shape[1]

#     if deviation_vecs is None:
#         # initialize (ndim_ps) deviation vectors
#         A = np.zeros((ndim_ps,ndim_ps))
#         for ii in range(ndim_ps):
#             A[ii] = np.random.normal(0.,1.,size=ndim_ps)
#             A[ii] /= np.linalg.norm(A[ii])

#         vec = gram_schmidt(A)
#         A = A[:2]

#     else:
#         raise NotImplementedError()

#     all_w0 = np.zeros((2,ndim_ps*2))
#     for ii in range(2):
#         all_w0[ii] = np.append(w0,A[ii])

#     # array to store the full, main orbit
#     full_w = np.zeros((nsteps+1,ndim_ps))
#     full_w[0] = w0
#     full_ts = np.zeros((nsteps+1,))
#     full_ts[0] = t1

#     # arrays to store the sali
#     sali = np.zeros((nsteps+1,))

#     time = t1
#     for ii in range(1,nsteps+1):
#         tt,ww = integrator.run(all_w0, dt=dt, nsteps=1, t1=time)
#         time += dt

#         dm = np.sqrt(np.sum((ww[-1,0,ndim_ps:] - ww[-1,1,ndim_ps:])**2))
#         dq = np.sqrt(np.sum((ww[-1,0,ndim_ps:] + ww[-1,1,ndim_ps:])**2))
#         sali[ii] = min(dm, dq)

#         # renormalize
#         ww[-1,0,ndim_ps:] = ww[-1,0,ndim_ps:] / np.linalg.norm(ww[-1,0,ndim_ps:])
#         ww[-1,1,ndim_ps:] = ww[-1,1,ndim_ps:] / np.linalg.norm(ww[-1,1,ndim_ps:])

#         full_w[ii:ii+1] = ww[-1,0,:ndim_ps]
#         full_ts[ii:ii+1] = time
#         all_w0 = ww[-1].copy()

#     return sali, full_ts, full_w
<|endoftext|>


<commit_before>#  Copyright (c) 2022. The Pycroft Authors. See the AUTHORS file.
#  This file is part of the Pycroft project and licensed under the terms of
#  the Apache License, Version 2.0. See the LICENSE file for details
from typing import NamedTuple

from sqlalchemy import and_, func, select, join, dialects
from sqlalchemy.orm import scoped_session, sessionmaker, joinedload, foreign

from ldap_sync import logger
from pycroft.model import create_engine
from pycroft.model.property import CurrentProperty
from pycroft.model.session import set_scoped_session, session as global_session
from pycroft.model.user import User, Group, Membership

def establish_and_return_session(connection_string):
    engine = create_engine(connection_string)
    set_scoped_session(scoped_session(sessionmaker(bind=engine)))
    return global_session  # from pycroft.model.session

class UserProxyType(NamedTuple):
    """Representation of a user as returned by :func:`fetch_users_to_sync`."""
    User: User
    should_be_blocked: bool

def fetch_users_to_sync(session, required_property=None) -> list[UserProxyType]:
    """Fetch the users who should be synced

    :param session: The SQLAlchemy session to use
    :param str required_property: the property required to export users

    :returns: An iterable of `(User, should_be_blocked)` ResultProxies
        having the property `required_property' and a unix_account.
    """
    if required_property:
        no_unix_account_q = User.q.join(User.current_properties)\
            .filter(CurrentProperty.property_name == required_property,
                    User.unix_account == None)
    else:
        no_unix_account_q = User.q.filter(User.unix_account == None)

    count_exportable_but_no_account = no_unix_account_q.count()

    if count_exportable_but_no_account:
        if required_property:
            logger.warning("%s users have the '%s' property but not a unix_account",
                           count_exportable_but_no_account, required_property)
        else:
            logger.warning("%s users applicable to exporting don't have a unix_account",
                           count_exportable_but_no_account)

    # used for second join against CurrentProperty
    not_blocked_property = CurrentProperty.__table__.alias('ldap_login_enabled')

    return (
        # Grab all users with the required property
        User.q
        .options(joinedload(User.unix_account))
        .join(User.current_properties)
        .filter((CurrentProperty.property_name == required_property) if required_property else True,
                User.unix_account != None)

        # additional info:
        #  absence of `ldap_login_enabled` property → should_be_blocked
        .add_columns(not_blocked_property.c.property_name.is_(None).label('should_be_blocked'))
        .outerjoin(
            not_blocked_property,
            and_(User.id == foreign(not_blocked_property.c.user_id),
                 ~not_blocked_property.c.denied,
                 not_blocked_property.c.property_name == 'ldap_login_enabled')
        ).all()
    )

class GroupProxyType(NamedTuple):
    """Representation of a group as returned by :func:`fetch_groups_to_sync`."""
    Group: Group
    members: list[str]

def fetch_groups_to_sync(session) -> list[GroupProxyType]:
    """Fetch the groups who should be synced

    :param session: The SQLAlchemy session to use

    :returns: An iterable of `(Group, members)` ResultProxies.
    """
    return (
        Group.q
        # uids of the members of the group
        .add_columns(func.coalesce(select(func.array_agg(User.login))
                .select_from(join(Membership, User))
                .where(Membership.group_id == Group.id)
                .where(Membership.active_during.contains(func.current_timestamp()))
                .group_by(Group.id)
                .scalar_subquery(),
                func.cast('{}', dialects.postgresql.ARRAY(User.login.type))).label('members'))
        .all()
    )

class PropertyProxyType(NamedTuple):
    """Representation of a property as returned by :func:`fetch_properties_to_sync`."""
    name: str
    members: list[str]

def fetch_properties_to_sync(session) -> list[PropertyProxyType]:
    """Fetch the groups who should be synced

    :param session: The SQLAlchemy session to use

    :returns: An iterable of `(property_name, members)` ResultProxies.
    """
    properties = session.execute(
        select(CurrentProperty.property_name.label('name'),
               func.array_agg(User.login).label('members'))
        .select_from(join(CurrentProperty, User, onclause=CurrentProperty.user_id == User.id))
        .where(CurrentProperty.property_name.in_(EXPORTED_PROPERTIES))
        .group_by(CurrentProperty.property_name)
    ).fetchall()

    missing_properties = EXPORTED_PROPERTIES - { p.name for p in properties }
    # Return mutable copy instead of SQLAlchemy's immutable RowProxy
    return [PropertyProxyType(p.name, p.members) for p in properties] \
           + [PropertyProxyType(p, []) for p in missing_properties]

EXPORTED_PROPERTIES = frozenset([
    'network_access',
    'mail',
    'traffic_limit_exceeded',
    'payment_in_default',
    'violation',
    'member',
    'ldap_login_enabled',
    'userdb',
    'cache_access',
    'membership_fee',
])
<commit_msg>Fix strict mypy issues in `ldap_sync.db`<commit_after>#  Copyright (c) 2022. The Pycroft Authors. See the AUTHORS file.
#  This file is part of the Pycroft project and licensed under the terms of
#  the Apache License, Version 2.0. See the LICENSE file for details
import typing
from typing import NamedTuple

from sqlalchemy import and_, func, select, join
from sqlalchemy.dialects import postgresql
from sqlalchemy.orm import scoped_session, sessionmaker, joinedload, foreign, Session

from ldap_sync import logger
from pycroft.model import create_engine
from pycroft.model.property import CurrentProperty
from pycroft.model.session import set_scoped_session, session as global_session
from pycroft.model.user import User, Group, Membership

def establish_and_return_session(connection_string: str) -> Session:
    engine = create_engine(connection_string)
    set_scoped_session(scoped_session(sessionmaker(bind=engine)))
    return typing.cast(Session, global_session)  # from pycroft.model.session

class UserProxyType(NamedTuple):
    """Representation of a user as returned by :func:`fetch_users_to_sync`."""
    User: User
    should_be_blocked: bool

def fetch_users_to_sync(
    session: Session, required_property: str | None = None
) -> list[UserProxyType]:
    """Fetch the users who should be synced

    :param session: The SQLAlchemy session to use
    :param str required_property: the property required to export users

    :returns: An iterable of `(User, should_be_blocked)` ResultProxies
        having the property `required_property' and a unix_account.
    """
    if required_property:
        no_unix_account_q = User.q.join(User.current_properties)\
            .filter(CurrentProperty.property_name == required_property,
                    User.unix_account == None)
    else:
        no_unix_account_q = User.q.filter(User.unix_account == None)

    count_exportable_but_no_account = no_unix_account_q.count()

    if count_exportable_but_no_account:
        if required_property:
            logger.warning("%s users have the '%s' property but not a unix_account",
                           count_exportable_but_no_account, required_property)
        else:
            logger.warning("%s users applicable to exporting don't have a unix_account",
                           count_exportable_but_no_account)

    # used for second join against CurrentProperty
    not_blocked_property = CurrentProperty.__table__.alias('ldap_login_enabled')

    return typing.cast(
        list[UserProxyType],
        # Grab all users with the required property
        User.q
        .options(joinedload(User.unix_account))
        .join(User.current_properties)
        .filter((CurrentProperty.property_name == required_property) if required_property else True,
                User.unix_account != None)

        # additional info:
        #  absence of `ldap_login_enabled` property → should_be_blocked
        .add_columns(not_blocked_property.c.property_name.is_(None).label('should_be_blocked'))
        .outerjoin(
            not_blocked_property,
            and_(User.id == foreign(not_blocked_property.c.user_id),
                 ~not_blocked_property.c.denied,
                 not_blocked_property.c.property_name == 'ldap_login_enabled')
        ).all()
    )

class GroupProxyType(NamedTuple):
    """Representation of a group as returned by :func:`fetch_groups_to_sync`."""
    Group: Group
    members: list[str]

def fetch_groups_to_sync(session: Session) -> list[GroupProxyType]:
    """Fetch the groups who should be synced

    :param session: The SQLAlchemy session to use

    :returns: An iterable of `(Group, members)` ResultProxies.
    """
    return typing.cast(
        list[GroupProxyType],
        Group.q
        # uids of the members of the group
        .add_columns(func.coalesce(select(func.array_agg(User.login))
                .select_from(join(Membership, User))
                .where(Membership.group_id == Group.id)
                .where(Membership.active_during.contains(func.current_timestamp()))
                .group_by(Group.id)
                .scalar_subquery(),
                func.cast("{}", postgresql.ARRAY(User.login.type)),
            ).label("members")
        ).all(),
    )

class PropertyProxyType(NamedTuple):
    """Representation of a property as returned by :func:`fetch_properties_to_sync`."""
    name: str
    members: list[str]

def fetch_properties_to_sync(session: Session) -> list[PropertyProxyType]:
    """Fetch the groups who should be synced

    :param session: The SQLAlchemy session to use

    :returns: An iterable of `(property_name, members)` ResultProxies.
    """
    properties = session.execute(
        select(CurrentProperty.property_name.label('name'),
               func.array_agg(User.login).label('members'))
        .select_from(join(CurrentProperty, User, onclause=CurrentProperty.user_id == User.id))
        .where(CurrentProperty.property_name.in_(EXPORTED_PROPERTIES))
        .group_by(CurrentProperty.property_name)
    ).fetchall()

    missing_properties = EXPORTED_PROPERTIES - { p.name for p in properties }
    # Return mutable copy instead of SQLAlchemy's immutable RowProxy
    return [PropertyProxyType(p.name, p.members) for p in properties] \
           + [PropertyProxyType(p, []) for p in missing_properties]

EXPORTED_PROPERTIES = frozenset([
    'network_access',
    'mail',
    'traffic_limit_exceeded',
    'payment_in_default',
    'violation',
    'member',
    'ldap_login_enabled',
    'userdb',
    'cache_access',
    'membership_fee',
])
<|endoftext|>


<commit_before>#!/usr/bin/python

import unittest
import sys
import traceback

import testcases
from vmpoolstate import VMPoolState
from dijkstra import VMPoolShortestPathFinder
from aspiers import VMPoolAdamPathFinder

#STRATEGY = VMPoolShortestPathFinder
STRATEGY = VMPoolAdamPathFinder

while True:
    stateA, stateB, expected_path = testcases.random.identical_hosts(2, 4)

    path_finder = STRATEGY(stateA, stateB)
    try:
        path = path_finder.find_path()
    except RuntimeError, exc:
        print path_finder.get_debug()
        traceback.print_exc(exc)
        sys.exit(1)

    if path:
        print path.summary()
    else:
        print path_finder.challenge_visualization(10, 80)
        print "\nNo path found to animate!"
        sys.exit(1)
<commit_msg>fix bug in soaktest when no path found<commit_after>#!/usr/bin/python

import unittest
import sys
import traceback

import testcases
from vmpoolstate import VMPoolState
from dijkstra import VMPoolShortestPathFinder
from aspiers import VMPoolAdamPathFinder

#STRATEGY = VMPoolShortestPathFinder
STRATEGY = VMPoolAdamPathFinder

while True:
    stateA, stateB, expected_path = testcases.random.identical_hosts(2, 4)

    path_finder = STRATEGY(stateA, stateB)
    try:
        path = path_finder.find_path()
    except RuntimeError, exc:
        print path_finder.get_debug()
        traceback.print_exc(exc)
        sys.exit(1)

    if path:
        print path.summary()
    else:
        print path_finder.path.challenge_visualization(10, 80)
        print "\nNo path found to animate!"
        sys.exit(1)
<|endoftext|>


<commit_before>import random
import string
from time import time

from sdk.codexbot_sdk import CodexBot
from config import APPLICATION_TOKEN, APPLICATION_NAME, DB, URL, SERVER

class Github:

    def __init__(self):

        self.sdk = CodexBot(APPLICATION_NAME, SERVER['host'], SERVER['port'], db_config=DB, token=APPLICATION_TOKEN)

        self.sdk.log("Github module initialized")

        self.sdk.register_commands([
            ('github_help', 'help', self.help),
            # ('github_start', 'start', self.start)
        ])

        self.sdk.set_routes([
            # ('POST', '/github/{user_token}', self.github_route_handler)
        ])

        self.sdk.start_server()

    #
    #
    # HELP
    # todo: move to the class
    #

    async def help(self, payload):
        self.sdk.log("/help handler fired with payload {}".format(payload))

        message = "Модуль для работы с сервисом GitHub.\n\n" \
                  "- Оповещения о новых Push-событиях\n" \
                  "- Оповещения о создании Pull-реквестов\n" \
                  "- Оповещения о создании Issues\n\n" \

        # TODO get repos list
        # repositories = list()
        #
        # if not repositories:
        #     message += "В данный момент модуль не активирован.\n\n" \
        #                "Для настройки модуля, используйте команду /github_start"
        # else:
        #     message += "Подключенные репозитории.\n\n"
        #     for repository in repositories:
        #         message += "{}\n".format(repository["name"])
        #     message += "\nДля отключения репозитория используйте команду /github_stop\n" \
        #            "Подключить еще один репозиторий можно с помощью команды /github_start\n\n" \
        #            "Меню модуля: /github_help"

        await self.sdk.send_to_chat(
            payload["chat"],
            message
        )

if __name__ == "__main__":
    github = Github()
<commit_msg>💂 add start command<commit_after>import random
import string
from time import time

from sdk.codexbot_sdk import CodexBot
from config import APPLICATION_TOKEN, APPLICATION_NAME, DB, URL, SERVER

class Github:

    def __init__(self):

        self.sdk = CodexBot(APPLICATION_NAME, SERVER['host'], SERVER['port'], db_config=DB, token=APPLICATION_TOKEN)

        self.sdk.log("Github module initialized")

        self.sdk.register_commands([
            ('github_help', 'help', self.help),
            ('github_start', 'start', self.start)
        ])

        self.sdk.set_routes([
            # ('POST', '/github/{user_token}', self.github_route_handler)
        ])

        self.sdk.start_server()

    #
    #
    # HELP
    # todo: move to the class
    #

    async def help(self, payload):
        self.sdk.log("/help handler fired with payload {}".format(payload))

        message = "Модуль для работы с сервисом GitHub.\n\n" \
                  "- Оповещения о новых Push-событиях\n" \
                  "- Оповещения о создании Pull-реквестов\n" \
                  "- Оповещения о создании Issues\n\n" \

        # TODO get repos list
        repositories = list()

        if not repositories:
            message += "В данный момент модуль не активирован.\n\n" \
                       "Для настройки модуля, используйте команду /github_start"
        # else:
        #     message += "Подключенные репозитории.\n\n"
        #     for repository in repositories:
        #         message += "{}\n".format(repository["name"])
        #     message += "\nДля отключения репозитория используйте команду /github_stop\n" \
        #            "Подключить еще один репозиторий можно с помощью команды /github_start\n\n" \
        #            "Меню модуля: /github_help"

        await self.sdk.send_to_chat(
            payload["chat"],
            message
        )

    async def start(self, payload):
        self.sdk.log("/help handler fired with payload {}".format(payload))

        message = "Чтобы подключить репозиторий, выполните следующие шаги.\n\n" \
                  "1) Откройте настройки вашего репозитория.\n\n" \
                  "2) Зайдите в раздел Webhooks & services и нажмите кнопку Add Webhook.\n\n" \
                  "3) Вставьте в поле Payload URL следующую ссылку.\n{link}\n\n" \
                  "4) В поле «Which events would you like to trigger this webhook?» выберите " \
                  "«Let me select individual events» и отметьте следующие флажки: \n" \
                  "- Issues\n- Pull request\n- Push\n\n" \
                  "5) В поле «Content type» выберите тип «application/json».\n\n" \
                  "6) Нажмите на кнопку «Add Webhook»."
        await self.sdk.send_to_chat(
            payload["chat"],
            message
        )

if __name__ == "__main__":
    github = Github()
<|endoftext|>