<commit_before>"""Create a mongoDB from the UMLS Rich Release Format files."""
import os
import langid
from io import open
import re

from pymongo import MongoClient
from collections import defaultdict
from pymongo.errors import CollectionInvalid
from tqdm import tqdm

PUNCT = re.compile("\W")

LANGDICT = {'ENG': 'en',
            'BAQ': 'eu',
            'CHI': 'zh',
            'CZE': 'cz',
            'DAN': 'dk',
            'DUT': 'nl',
            'EST': 'et',
            'FIN': 'fi',
            'FRE': 'fr',
            'GER': 'de',
            'GRE': 'gr',
            'HEB': 'he',
            'HUN': 'hu',
            'ITA': 'it',
            'JPN': 'jp',
            'KOR': 'ko',
            'LAV': 'lv',
            'NOR': 'no',
            'POL': 'pl',
            'POR': 'po',
            'RUS': 'ru',
            'SPA': 'sp',
            'SWE': 'sw',
            'TUR': 'tr'}

RELATIONMAPPING = {"PAR": 'parent',
                   "CHD": 'child',
                   "RB": 'broader',
                   "RN": 'narrower',
                   "SY": 'synonym',
                   "RO": 'other',
                   "RL": 'similar',
                   "RQ": 'related',
                   "SIB": 'sibling',
                   "AQ": 'qualifier',
                   "QB": 'qualifies',
                   "RU": 'unspecified',
                   "XR": 'notrelated'}

def createdb(pathtometadir,
             languages=(),
             dbname="umls",
             host="localhost",
             port=27017,
             process_definitions=True,
             process_relations=True,
             process_semantic_types=True,
             preprocessor=lambda x: x,
             overwrite=False):
    """
    Create a MongoDB instance from the RRF format in which UMLS is distributed.

    The class is designed to directly operate on the RRF files,
    and doesn't require any formatting.
    It will create a mongoDB Database with the pre-specified name,
    host and port.

    Parameters
    ----------
    languages : list of string
        The languages to extract from the UMLS database. These must be
        passed in UMLS format, not in an ISO-compliant standard.
        See LANGDICT for language key value mappings.
    pathtometadir : string
        The path to the META directory which contains the RRF files. The
        only files that Humumls needs are the MRCONSO, MRDEF and MRREL
        files.
    dbname : string, optional, default "umls"
        The name of the mongodb database you are about to create.
    host : string, optional, default "localhost"
        The name of your host.
    port : int
        The port on which your mongodb instance resides.
    preprocessor : function, optional, default identity
        The preprocessor you would like to use. This should be a function
        which takes a string and returns a string. An example of this is
        a tokenizer or a HTML stripper.

    """
    client = MongoClient(host=host, port=port)
    db = client.get_database(dbname)

    # Remove any duplicates
    languages = set(languages)

    # Transform non-standard language codings to ISO.
    try:
        {LANGDICT[l.upper()] for l in languages}
    except KeyError:
        raise KeyError("Not all languages you passed are valid.")
    # First create necessary paths, to fail early.
    try:
        collection = db.create_collection("term")
        terms = _create_terms(pathtometadir, languages)
        collection.insert_many(terms)
        del(terms)
    except CollectionInvalid:
        if overwrite:
            db.drop_collection("term")
            collection = db.create_collection("term")
            terms = _create_terms(pathtometadir, languages)
            collection.insert_many(terms)
            del(terms)
        else:
            print("term already exists, not overwriting.")

    try:
        collection = db.create_collection("string")
        strings = _create_strings(pathtometadir, languages)
        collection.insert_many(strings)
        del(strings)
    except CollectionInvalid:
        if overwrite:
            db.drop_collection("string")
            collection = db.create_collection("string")
            strings = _create_strings(pathtometadir, languages)
            collection.insert_many(strings)
            del(strings)
        else:
            print("string already exists, not overwriting.")

    try:
        collection = db.create_collection("concept")
        concepts = _create_concepts(pathtometadir,
                                    process_definitions,
                                    process_relations,
                                    process_semantic_types,
                                    languages,
                                    preprocessor)
        collection.insert_many(concepts)
        del(concepts)
    except CollectionInvalid:
        if overwrite:
            collection = db.create_collection("concept")
            concepts = _create_concepts(pathtometadir,
                                        process_definitions,
                                        process_relations,
                                        process_semantic_types,
                                        languages,
                                        preprocessor)
            collection.insert_many(concepts)
            del(concepts)
        else:
            print("concept already exists, not overwriting.")

    return db

def _create_concepts(path,
                     process_definitions,
                     process_relations,
                     process_semantic_types,
                     languages,
                     preprocessor):
    """
    Read MRCONSO for concepts.

    Parameters
    ----------
    path : string
        The path to the folder containing the MRCONSO file.
    process_definitions : bool
        Whether to process MRDEF, and add definitions to the database.
    process_relations : bool
        Whether to process MRREL, and add relations to the database.
    process_semantic_types : bool
        Whether to process MRSTY, and add semantic types to the database.
    languages : list of str
        The languages to use.
    preprocessor : function
        A function which preprocesses the data.

    Returns
    -------
    concepts : dict
        Dictionary of concept data, to be added to the database.

    """
    concepts = defaultdict(dict)

    mrcsonsopath = os.path.join(path, "MRCONSO.RRF")

    for idx, _ in enumerate(open(mrcsonsopath)):
        pass

    num_lines = idx
    print("Reading MRCONSO for concepts.")
    for record in tqdm(open(mrcsonsopath), total=num_lines):
        split = record.strip().split("|")

        if languages and split[1] not in languages:
            continue

        cui = split[0]
        sui = split[5]
        lui = split[3]

        c = concepts[cui]

        c["_id"] = cui

        if split[2] == "P":
            c["preferred"] = lui

        try:
            c["lui"].add(lui)
        except KeyError:
            c["lui"] = set([lui])
        try:
            c["sui"].add(sui)
        except KeyError:
            c["sui"] = set([sui])

    if process_definitions:
        concepts = process_mrdef(path, concepts, languages, preprocessor)
    if process_relations:
        concepts = process_mrrel(path, concepts)
    if process_semantic_types:
        concepts = process_mrsty(path, concepts)

    for v in concepts.values():
        try:
            v['lui'] = list(v['lui'])
        except KeyError:
            pass
        try:
            v['sui'] = list(v['sui'])
        except KeyError:
            pass

    return list(concepts.values())

def _create_terms(path, languages):
    """Read MRCONSO for terms."""
    terms = defaultdict(dict)

    mrcsonsopath = os.path.join(path, "MRCONSO.RRF")

    for idx, _ in enumerate(open(mrcsonsopath)):
        pass

    num_lines = idx

    print("Reading MRCONSO for terms.")
    for record in tqdm(open(mrcsonsopath), total=num_lines):

        split = record.strip().split("|")

        if languages and split[1] not in languages:
            continue

        cui = split[0]
        sui = split[5]
        lui = split[3]

        t = terms[lui]

        t["_id"] = lui
        try:
            t["cui"].append(cui)
        except KeyError:
            t["cui"] = [cui]
        try:
            t["sui"].append(sui)
        except KeyError:
            t["sui"] = [sui]

    for v in terms.values():
        t["sui"] = list(t["sui"])
        t["cui"] = list(t["cui"])

    return list(terms.values())

def _create_strings(path, languages):
    """Read MRCONSO for strings."""
    strings = defaultdict(dict)

    mrcsonsopath = os.path.join(path, "MRCONSO.RRF")

    for idx, _ in enumerate(open(mrcsonsopath)):
        pass

    num_lines = idx

    print("Reading MRCONSO for strings.")
    for record in tqdm(open(mrcsonsopath), total=num_lines):

        split = record.strip().split("|")
        string = split[14]

        # Check BSON length
        byte_string = string.encode("utf-8")
        if len(byte_string) >= 1000:
            # Truncate 1000 bytes
            string = byte_string[:1000].decode('utf-8')

        if languages and split[1] not in languages:
            continue

        cui = split[0]
        sui = split[5]
        lui = split[3]

        # Create lexical representation.
        lowerwords = " ".join(PUNCT.sub(" ", string).lower().split())

        s = strings[sui]

        s["_id"] = sui
        s["string"] = string
        s["lower"] = lowerwords
        s["lang"] = split[1]
        s["numwords"] = len(string.split())
        s["numwordslower"] = len(lowerwords.split())
        s["lui"] = lui
        try:
            s["cui"].add(cui)
        except KeyError:
            s["cui"] = set([cui])

    for v in strings.values():
        v['cui'] = list(v['cui'])

    return list(strings.values())

def process_mrrel(path, concepts):
    """
    Read the relations from MRREL.RRF, and add them to concepts.

    Because bidirectional relations in UMLS occur for both directions,
    only the direction which occurs in the UMLS is added.
    """
    mrrelpath = os.path.join(path, "MRREL.RRF")

    for idx, _ in enumerate(open(mrrelpath)):
        pass

    num_lines = idx

    print("Reading MRREL.RRF for relations.")
    for record in tqdm(open(mrrelpath), total=num_lines):

        split = record.strip().split("|")

        cui = split[4]
        dest = split[0]

        # provide dictionary mapping for REL
        rel = RELATIONMAPPING[split[3]]

        c = concepts[cui]
        c["rel"] = c.get("rel", {})

        try:
            concepts[cui]["rel"][rel].add(dest)
        except KeyError:
            concepts[cui]["rel"][rel] = set([dest])

    for v in concepts.values():
        try:
            for reltype in v["rel"]:
                v["rel"][reltype] = list(v["rel"][reltype])
        except KeyError:
            pass

    return concepts

def process_mrdef(path,
                  concepts,
                  languages,
                  preprocessor):
    """
    Read definitions from MRDEF.RRF.

    We append this to the intermediate dictionary (concepts), not the DB
    because this is way faster.

    Parameters
    ----------
    path : string
        The path to the META dir.
    concepts : dict
        The dictionary of concepts to which to add the definitions.
    languages : list of str
        The languages to use.
    preprocessor : function
        Function that preprocesses the defintions. Should be a function which
        takes a string as input and returns a string.

    Returns
    -------
    concepts : dict
        The updated concept dictionary with added definitions.

    """
    isolanguages = {LANGDICT[l.upper()] for l in languages}
    print(isolanguages)

    mrdefpath = os.path.join(path, "MRDEF.RRF")

    for idx, _ in enumerate(open(mrdefpath)):
        pass

    num_lines = idx

    print("Reading MRDEF.RRF for definitions.")
    for record in tqdm(open(mrdefpath), total=num_lines):
        split = record.strip().split("|")

        cui = split[0]
        c = concepts[cui]
        definition = split[5]

        # Detect language -> UMLS does not take into account language
        # in MRDEF.
        lang, _ = langid.classify(definition)
        if lang not in isolanguages:
            continue

        # Tokenize the definition.
        if preprocessor:
            definition = preprocessor(definition)
        try:
            c["definition"].append(definition)
        except KeyError:
            c["definition"] = [definition]

    return concepts

def process_mrsty(path, concepts):
    """Read semantic types from MRSTY.RRF."""
    mrstypath = os.path.join(path, "MRSTY.RRF")

    for idx, _ in enumerate(open(mrstypath)):
        pass

    num_lines = idx

    print("Reading MRSTY.RRF for semantic types.")
    for record in tqdm(open(mrstypath), total=num_lines):
        split = record.strip().split("|")

        cui = split[0]
        if cui not in concepts:
            continue
        c = concepts[cui]
        semantic_type = split[2]
        try:
            c = c["semtype"].append(semantic_type)
        except KeyError:
            c["semtype"] = [semantic_type]

    return concepts
<commit_msg>Make tablecreator more principled by not lowercasing<commit_after>"""Create a mongoDB from the UMLS Rich Release Format files."""
import os
import langid
from io import open
import re

from pymongo import MongoClient
from collections import defaultdict
from pymongo.errors import CollectionInvalid
from tqdm import tqdm

PUNCT = re.compile("\W")

LANGDICT = {'ENG': 'en',
            'BAQ': 'eu',
            'CHI': 'zh',
            'CZE': 'cz',
            'DAN': 'dk',
            'DUT': 'nl',
            'EST': 'et',
            'FIN': 'fi',
            'FRE': 'fr',
            'GER': 'de',
            'GRE': 'gr',
            'HEB': 'he',
            'HUN': 'hu',
            'ITA': 'it',
            'JPN': 'jp',
            'KOR': 'ko',
            'LAV': 'lv',
            'NOR': 'no',
            'POL': 'pl',
            'POR': 'po',
            'RUS': 'ru',
            'SPA': 'sp',
            'SWE': 'sw',
            'TUR': 'tr'}

RELATIONMAPPING = {"PAR": 'parent',
                   "CHD": 'child',
                   "RB": 'broader',
                   "RN": 'narrower',
                   "SY": 'synonym',
                   "RO": 'other',
                   "RL": 'similar',
                   "RQ": 'related',
                   "SIB": 'sibling',
                   "AQ": 'qualifier',
                   "QB": 'qualifies',
                   "RU": 'unspecified',
                   "XR": 'notrelated'}

def createdb(pathtometadir,
             languages=(),
             dbname="umls",
             host="localhost",
             port=27017,
             process_definitions=True,
             process_relations=True,
             process_semantic_types=True,
             preprocessor=lambda x: x,
             overwrite=False):
    """
    Create a MongoDB instance from the RRF format in which UMLS is distributed.

    The class is designed to directly operate on the RRF files,
    and doesn't require any formatting.
    It will create a mongoDB Database with the pre-specified name,
    host and port.

    Parameters
    ----------
    languages : list of string
        The languages to extract from the UMLS database. These must be
        passed in UMLS format, not in an ISO-compliant standard.
        See LANGDICT for language key value mappings.
    pathtometadir : string
        The path to the META directory which contains the RRF files. The
        only files that Humumls needs are the MRCONSO, MRDEF and MRREL
        files.
    dbname : string, optional, default "umls"
        The name of the mongodb database you are about to create.
    host : string, optional, default "localhost"
        The name of your host.
    port : int
        The port on which your mongodb instance resides.
    preprocessor : function, optional, default identity
        The preprocessor you would like to use. This should be a function
        which takes a string and returns a string. An example of this is
        a tokenizer or a HTML stripper.

    """
    client = MongoClient(host=host, port=port)
    db = client.get_database(dbname)

    # Remove any duplicates
    languages = set(languages)

    # Transform non-standard language codings to ISO.
    try:
        {LANGDICT[l.upper()] for l in languages}
    except KeyError:
        raise KeyError("Not all languages you passed are valid.")
    # First create necessary paths, to fail early.
    try:
        collection = db.create_collection("term")
        terms = _create_terms(pathtometadir, languages)
        collection.insert_many(terms)
        del(terms)
    except CollectionInvalid:
        if overwrite:
            db.drop_collection("term")
            collection = db.create_collection("term")
            terms = _create_terms(pathtometadir, languages)
            collection.insert_many(terms)
            del(terms)
        else:
            print("term already exists, not overwriting.")

    try:
        collection = db.create_collection("string")
        strings = _create_strings(pathtometadir, languages)
        collection.insert_many(strings)
        del(strings)
    except CollectionInvalid:
        if overwrite:
            db.drop_collection("string")
            collection = db.create_collection("string")
            strings = _create_strings(pathtometadir, languages)
            collection.insert_many(strings)
            del(strings)
        else:
            print("string already exists, not overwriting.")

    try:
        collection = db.create_collection("concept")
        concepts = _create_concepts(pathtometadir,
                                    process_definitions,
                                    process_relations,
                                    process_semantic_types,
                                    languages,
                                    preprocessor)
        collection.insert_many(concepts)
        del(concepts)
    except CollectionInvalid:
        if overwrite:
            collection = db.create_collection("concept")
            concepts = _create_concepts(pathtometadir,
                                        process_definitions,
                                        process_relations,
                                        process_semantic_types,
                                        languages,
                                        preprocessor)
            collection.insert_many(concepts)
            del(concepts)
        else:
            print("concept already exists, not overwriting.")

    return db

def _create_concepts(path,
                     process_definitions,
                     process_relations,
                     process_semantic_types,
                     languages,
                     preprocessor):
    """
    Read MRCONSO for concepts.

    Parameters
    ----------
    path : string
        The path to the folder containing the MRCONSO file.
    process_definitions : bool
        Whether to process MRDEF, and add definitions to the database.
    process_relations : bool
        Whether to process MRREL, and add relations to the database.
    process_semantic_types : bool
        Whether to process MRSTY, and add semantic types to the database.
    languages : list of str
        The languages to use.
    preprocessor : function
        A function which preprocesses the data.

    Returns
    -------
    concepts : dict
        Dictionary of concept data, to be added to the database.

    """
    concepts = defaultdict(dict)

    mrcsonsopath = os.path.join(path, "MRCONSO.RRF")

    for idx, _ in enumerate(open(mrcsonsopath)):
        pass

    num_lines = idx
    print("Reading MRCONSO for concepts.")
    for record in tqdm(open(mrcsonsopath), total=num_lines):
        split = record.strip().split("|")

        if languages and split[1] not in languages:
            continue

        cui = split[0]
        sui = split[5]
        lui = split[3]

        c = concepts[cui]

        c["_id"] = cui

        if split[2] == "P":
            c["preferred"] = lui

        try:
            c["lui"].add(lui)
        except KeyError:
            c["lui"] = set([lui])
        try:
            c["sui"].add(sui)
        except KeyError:
            c["sui"] = set([sui])

    if process_definitions:
        concepts = process_mrdef(path, concepts, languages, preprocessor)
    if process_relations:
        concepts = process_mrrel(path, concepts)
    if process_semantic_types:
        concepts = process_mrsty(path, concepts)

    for v in concepts.values():
        try:
            v['lui'] = list(v['lui'])
        except KeyError:
            pass
        try:
            v['sui'] = list(v['sui'])
        except KeyError:
            pass

    return list(concepts.values())

def _create_terms(path, languages):
    """Read MRCONSO for terms."""
    terms = defaultdict(dict)

    mrcsonsopath = os.path.join(path, "MRCONSO.RRF")

    for idx, _ in enumerate(open(mrcsonsopath)):
        pass

    num_lines = idx

    print("Reading MRCONSO for terms.")
    for record in tqdm(open(mrcsonsopath), total=num_lines):

        split = record.strip().split("|")

        if languages and split[1] not in languages:
            continue

        cui = split[0]
        sui = split[5]
        lui = split[3]

        t = terms[lui]

        t["_id"] = lui
        try:
            t["cui"].append(cui)
        except KeyError:
            t["cui"] = [cui]
        try:
            t["sui"].append(sui)
        except KeyError:
            t["sui"] = [sui]

    for v in terms.values():
        t["sui"] = list(t["sui"])
        t["cui"] = list(t["cui"])

    return list(terms.values())

def _create_strings(path, languages):
    """Read MRCONSO for strings."""
    strings = defaultdict(dict)

    mrcsonsopath = os.path.join(path, "MRCONSO.RRF")

    for idx, _ in enumerate(open(mrcsonsopath)):
        pass

    num_lines = idx

    print("Reading MRCONSO for strings.")
    for record in tqdm(open(mrcsonsopath), total=num_lines):

        split = record.strip().split("|")
        string = split[14]

        # Check BSON length
        byte_string = string.encode("utf-8")
        if len(byte_string) >= 1000:
            # Truncate 1000 bytes
            string = byte_string[:1000].decode('utf-8')

        if languages and split[1] not in languages:
            continue

        cui = split[0]
        sui = split[5]
        lui = split[3]

        # Create lexical representation.
        tokenized = " ".join(PUNCT.sub(" ", string).split())

        s = strings[sui]

        s["_id"] = sui
        s["string"] = string
        s["lower"] = string.lower()
        s["tokenized"] = tokenized
        s["lang"] = split[1]
        s["numwords"] = len(string.split())
        s["numwordslower"] = len(tokenized.split())
        s["lui"] = lui
        try:
            s["cui"].add(cui)
        except KeyError:
            s["cui"] = set([cui])

    for v in strings.values():
        v['cui'] = list(v['cui'])

    return list(strings.values())

def process_mrrel(path, concepts):
    """
    Read the relations from MRREL.RRF, and add them to concepts.

    Because bidirectional relations in UMLS occur for both directions,
    only the direction which occurs in the UMLS is added.
    """
    mrrelpath = os.path.join(path, "MRREL.RRF")

    for idx, _ in enumerate(open(mrrelpath)):
        pass

    num_lines = idx

    print("Reading MRREL.RRF for relations.")
    for record in tqdm(open(mrrelpath), total=num_lines):

        split = record.strip().split("|")

        cui = split[4]
        dest = split[0]

        # provide dictionary mapping for REL
        rel = RELATIONMAPPING[split[3]]

        c = concepts[cui]
        c["rel"] = c.get("rel", {})

        try:
            concepts[cui]["rel"][rel].add(dest)
        except KeyError:
            concepts[cui]["rel"][rel] = set([dest])

    for v in concepts.values():
        try:
            for reltype in v["rel"]:
                v["rel"][reltype] = list(v["rel"][reltype])
        except KeyError:
            pass

    return concepts

def process_mrdef(path,
                  concepts,
                  languages,
                  preprocessor):
    """
    Read definitions from MRDEF.RRF.

    We append this to the intermediate dictionary (concepts), not the DB
    because this is way faster.

    Parameters
    ----------
    path : string
        The path to the META dir.
    concepts : dict
        The dictionary of concepts to which to add the definitions.
    languages : list of str
        The languages to use.
    preprocessor : function
        Function that preprocesses the defintions. Should be a function which
        takes a string as input and returns a string.

    Returns
    -------
    concepts : dict
        The updated concept dictionary with added definitions.

    """
    isolanguages = {LANGDICT[l.upper()] for l in languages}
    print(isolanguages)

    mrdefpath = os.path.join(path, "MRDEF.RRF")

    for idx, _ in enumerate(open(mrdefpath)):
        pass

    num_lines = idx

    print("Reading MRDEF.RRF for definitions.")
    for record in tqdm(open(mrdefpath), total=num_lines):
        split = record.strip().split("|")

        cui = split[0]
        c = concepts[cui]
        definition = split[5]

        # Detect language -> UMLS does not take into account language
        # in MRDEF.
        lang, _ = langid.classify(definition)
        if lang not in isolanguages:
            continue

        # Tokenize the definition.
        if preprocessor:
            definition = preprocessor(definition)
        try:
            c["definition"].append(definition)
        except KeyError:
            c["definition"] = [definition]

    return concepts

def process_mrsty(path, concepts):
    """Read semantic types from MRSTY.RRF."""
    mrstypath = os.path.join(path, "MRSTY.RRF")

    for idx, _ in enumerate(open(mrstypath)):
        pass

    num_lines = idx

    print("Reading MRSTY.RRF for semantic types.")
    for record in tqdm(open(mrstypath), total=num_lines):
        split = record.strip().split("|")

        cui = split[0]
        if cui not in concepts:
            continue
        c = concepts[cui]
        semantic_type = split[2]
        try:
            c = c["semtype"].append(semantic_type)
        except KeyError:
            c["semtype"] = [semantic_type]

    return concepts
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Copyright (C) 2018 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import argparse
import subprocess
import re

from datetime import datetime, timedelta, time as type_time, date as type_date
from subprocess import check_output, CalledProcessError, STDOUT
from vyos.util import ask_yes_no

def valid_time(s):
  try:
    return datetime.strptime(s, "%H:%M").time()
  except ValueError:
    return None

def valid_date(s):
  try:
    return datetime.strptime(s, "%d%m%Y").date()
  except ValueError:
    try:
      return datetime.strptime(s, "%d/%m/%Y").date()
    except ValueError:
      try:
        return datetime.strptime(s, "%d.%m.%Y").date()
      except ValueError:
        try:
          return datetime.strptime(s, "%d:%m:%Y").date()
        except ValueError:
          return None

def check_shutdown():
  try:
    cmd = check_output(["/bin/systemctl","status","systemd-shutdownd.service"])
    #Shutodwn is scheduled
    r = re.findall(r'Status: \"(.*)\"\n', cmd.decode())[0]
    print(r)
  except CalledProcessError as e:
    #Shutdown is not scheduled
    print("Shutdown is not scheduled")

def cancel_shutdown():
  try:
    timenow = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    cmd = check_output(["/sbin/shutdown","-c","--no-wall"])
    message = "Reboot scheduled has been cancelled %s" % timenow
    #Generate broadcast message about cancel reboot
    os.system("wall %s" % message)
  except CalledProcessError as e:
    sys.exit("Error aborting shutdown: %s" % e)

def execute_shutdown(time, reboot = True, ask=True):
  if not ask:
    action = "reboot" if reboot else "poweroff"
    if not ask_yes_no("Are you sure you want to %s this system?" % action):
      sys.exit(0)

  action = "-r" if reboot else "-P"

  if len(time) == 0:
    ### T870 legacy reboot job support
    chk_vyatta_based_reboots()
    ###

    cmd = check_output(["/sbin/shutdown",action,"now"],stderr=STDOUT)
    print(cmd.decode().split(",",1)[0])
    return

  # Try to extract date from the first argument
  if len(time) == 1:
    time = time[0].split(" ",1)

  if len(time) == 1:
    ts = valid_time(time[0])
    if time[0].isdigit() or valid_time(time[0]):
      cmd = check_output(["/sbin/shutdown",action,time[0]],stderr=STDOUT)
    else:
      sys.exit("Timestamp needs to be in format of 12:34")

  elif len(time) == 2:
    ts = valid_time(time[0])
    ds = valid_date(time[1])
    if ts and ds:
      t = datetime.combine(ds, ts)
      td = t - datetime.now()
      t2 = 1 + int(td.total_seconds())//60 # Get total minutes
      cmd = check_output(["/sbin/shutdown",action,str(t2)],stderr=STDOUT)
    else:
      sys.exit("Timestamp needs to be in format of 12:34\nDatestamp in the format of DD.MM.YY")
  else:
    sys.exit("Could not decode time and date")

  check_shutdown()

def chk_vyatta_based_reboots():
  ### T870 commit-confirm is still using the vyatta code base, once gone, the code below can be removed
  ### legacy scheduled reboot s are using at and store the is as /var/run/<name>.job
  ### name is the node of scheduled the job, commit-confirm checks for that

  f = r'/var/run/confirm.job'
  if os .path.exists(f):
    jid = open(f).read().strip()
    if jid != 0:
      subprocess.call(['sudo', 'atrm', jid])
    os.remove(f)

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument("--yes", "-y",
            help="dont as for shutdown",
            action="store_true",
            dest="yes")
  action = parser.add_mutually_exclusive_group(required=True)
  action.add_argument("--reboot", "-r",
            help="Reboot the system",
            nargs="*",
            metavar="Minutes|HH:MM")

  action.add_argument("--poweroff", "-p",
            help="Poweroff the system",
            nargs="*",
            metavar="Minutes|HH:MM")

  action.add_argument("--cancel", "-c",
            help="Cancel pending shutdown",
            action="store_true")

  action.add_argument("--check",
            help="Check pending chutdown",
            action="store_true")
  args = parser.parse_args()

  try:
    if  args.reboot is not None:
      execute_shutdown(args.reboot, reboot=True, ask=args.yes)
    if args.poweroff is not None:
      execute_shutdown(args.poweroff, reboot=False,ask=args.yes)
    if args.cancel:
      cancel_shutdown()
    if args.check:
      check_shutdown()
  except KeyboardInterrupt:
    sys.exit("Interrupted")

if __name__ == "__main__":
  main()
<commit_msg>T1855, T1826: clean up the reboot/shutdown script.<commit_after>#!/usr/bin/env python3
#
# Copyright (C) 2018 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import sys
import argparse
import subprocess
import re

from datetime import datetime, timedelta, time as type_time, date as type_date
from subprocess import check_output, CalledProcessError, STDOUT
from vyos.util import ask_yes_no

def parse_time(s):
  try:
    return datetime.strptime(s, "%H:%M").time()
  except ValueError:
    return None

def parse_date(s):
  for fmt in ["%d%m%Y", "%d/%m/%Y", "%d.%m.%Y", "%d:%m:%Y", "%Y-%m-%d"]:
    try:
      return datetime.strptime(s, fmt).date()
    except ValueError:
      continue
  # If nothing matched...
  return None

def get_shutdown_status():
  try:
    output = check_output(["/bin/systemctl", "status", "systemd-shutdownd.service"]).decode()
    return output
  except CalledProcessError:
    return None

def check_shutdown():
  output = get_shutdown_status()
  if output:
    r = re.findall(r'Status: \"(.*)\"\n', output)
    if r:
        # When available, that line is like
        # Status: "Shutting down at Thu 1970-01-01 00:00:00 UTC (poweroff)..."
        print(r[0])
    else:
        # Sometimes status string is not available immediately
        # after service startup
        print("Poweroff or reboot is scheduled")
  else:
    print("Poweroff or reboot is not scheduled")

def cancel_shutdown():
  output = get_shutdown_status()
  if output:
    try:
      timenow = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
      cmd = check_output(["/sbin/shutdown","-c","--no-wall"])
      message = "Scheduled reboot or poweroff has been cancelled %s" % timenow
      os.system("wall %s" % message)
    except CalledProcessError as e:
      sys.exit("Could not cancel a reboot or poweroff: %s" % e)
  else:
    print("Reboot or poweroff is not scheduled")

def execute_shutdown(time, reboot = True, ask=True):
  if not ask:
    action = "reboot" if reboot else "poweroff"
    if not ask_yes_no("Are you sure you want to %s this system?" % action):
      sys.exit(0)

  action = "-r" if reboot else "-P"

  if len(time) == 0:
    ### T870 legacy reboot job support
    chk_vyatta_based_reboots()
    ###

    cmd = check_output(["/sbin/shutdown",action,"now"],stderr=STDOUT)
    print(cmd.decode().split(",",1)[0])
    return
  elif len(time) == 1:
    # Assume the argument is just time
    ts = parse_time(time[0])
    if ts:
      cmd = check_output(["/sbin/shutdown", action, time[0]], stderr=STDOUT)
    else:
      sys.exit("Invalid time \"{0}\". The valid format is HH:MM".format(time[0]))
  elif len(time) == 2:
    # Assume it's date and time
    ts = parse_time(time[0])
    ds = parse_date(time[1])
    if ts and ds:
      t = datetime.combine(ds, ts)
      td = t - datetime.now()
      t2 = 1 + int(td.total_seconds())//60 # Get total minutes
      cmd = check_output(["/sbin/shutdown", action, str(t2)], stderr=STDOUT)
    else:
      if not ts:
        sys.exit("Invalid time \"{0}\". The valid format is HH:MM".format(time[0]))
      else:
        sys.exit("Invalid time \"{0}\". A valid format is YYYY-MM-DD [HH:MM]".format(time[1]))
  else:
    sys.exit("Could not decode date and time. Valids formats are HH:MM or YYYY-MM-DD HH:MM")
  check_shutdown()

def chk_vyatta_based_reboots():
  ### T870 commit-confirm is still using the vyatta code base, once gone, the code below can be removed
  ### legacy scheduled reboot s are using at and store the is as /var/run/<name>.job
  ### name is the node of scheduled the job, commit-confirm checks for that

  f = r'/var/run/confirm.job'
  if os.path.exists(f):
    jid = open(f).read().strip()
    if jid != 0:
      subprocess.call(['sudo', 'atrm', jid])
    os.remove(f)

def main():
  parser = argparse.ArgumentParser()
  parser.add_argument("--yes", "-y",
            help="Do not ask for confirmation",
            action="store_true",
            dest="yes")
  action = parser.add_mutually_exclusive_group(required=True)
  action.add_argument("--reboot", "-r",
            help="Reboot the system",
            nargs="*",
            metavar="Minutes|HH:MM")

  action.add_argument("--poweroff", "-p",
            help="Poweroff the system",
            nargs="*",
            metavar="Minutes|HH:MM")

  action.add_argument("--cancel", "-c",
            help="Cancel pending shutdown",
            action="store_true")

  action.add_argument("--check",
            help="Check pending chutdown",
            action="store_true")
  args = parser.parse_args()

  try:
    if  args.reboot is not None:
      execute_shutdown(args.reboot, reboot=True, ask=args.yes)
    if args.poweroff is not None:
      execute_shutdown(args.poweroff, reboot=False,ask=args.yes)
    if args.cancel:
      cancel_shutdown()
    if args.check:
      check_shutdown()
  except KeyboardInterrupt:
    sys.exit("Interrupted")

if __name__ == "__main__":
  main()

<|endoftext|>


<commit_before>#!/usr/bin/env python3
#
# Check PTL/liaison has approved release
#
# Copyright 2019 Thierry Carrez <thierry@openstack.org>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import argparse
import json
import logging
import os
import sys

import requests
from requests.packages import urllib3
import yaml

PROJECTS_YAML = 'reference/projects.yaml'
GERRIT_URL = 'https://review.opendev.org/'
LOG = logging.getLogger(__name__)

# Turn of warnings about bad SSL config.
# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
urllib3.disable_warnings()

def get_team(workspace, deliverablefile):
    with open(os.path.join(workspace, deliverablefile), 'r') as dfile:
        team = yaml.safe_load(dfile)['team']
    return team

def get_liaisons(workspace, team):
    filename = os.path.join(workspace, 'data/release_liaisons.yaml')
    with open(filename, 'r') as lfile:
        liaisons = yaml.safe_load(lfile)
    if team in liaisons:
        return [i['email'] for i in liaisons[team]]
    else:
        print('WARNING: %s team does not exist in liaisons file' % team)
        return []

class GerritChange(object):

    def __init__(self, args):
        # Load governance data
        with open(os.path.join(args.governance, PROJECTS_YAML), 'r') as dfile:
            self.gov_data = yaml.safe_load(dfile)

        try:
            self.load_from_gerrit(args.changeid)
        except KeyError:
            LOG.warning(
                '\ndata from gerrit is missing required keys:\n\n%s\n',
                json.dumps(self.raw_data, indent=2))
            raise

        self.workspace = args.releases

    def load_from_gerrit(self, changeid):
        # Grab changeid details from Gerrit
        call = 'changes/%s/detail' % changeid + \
               '?o=CURRENT_REVISION&o=CURRENT_FILES'
        raw = requests.get(GERRIT_URL + call)

        # Gerrit's REST API prepends a JSON-breaker to avoid XSS
        if raw.text.startswith(")]}'"):
            trimmed = raw.text[4:]
        else:
            trimmed = raw.text

        # Try to decode and bail with much detail if it fails
        try:
            decoded = json.loads(trimmed)
        except Exception:
            LOG.error(
                '\nrequest returned %s error to query:\n\n    %s\n'
                '\nwith detail:\n\n    %s\n',
                raw, raw.url, trimmed)
            raise

        # Instantiate object with retrieved data
        self.raw_data = decoded
        self.approvers = [i['email']
                          for i in decoded['labels']['Code-Review']['all']
                          if i['value'] > 0]
        self.approvers.append(decoded['owner']['email'])
        currev = decoded['current_revision']
        self.deliv_files = [
            x for x in decoded['revisions'][currev]['files'].keys()
            if x.startswith('deliverables/')
        ]

    def is_approved(self):
        LOG.debug('Approvals: %s' % self.approvers)
        approved = True
        for deliv_file in self.deliv_files:
            team = get_team(self.workspace, deliv_file)
            try:
                govteam = self.gov_data[team]
            except ValueError:
                print('✕ %s mentions unknown team %s' % (deliv_file, team))
                approved = False
                break

            # Check that deliverable is indeed defined in governance team
            delivname, _ = os.path.splitext(os.path.basename(deliv_file))
            if delivname not in govteam['deliverables']:
                print('✕ %s not in %s governance' % (deliv_file, team))
                approved = False
                break

            # Fetch PTL and release liaisons
            liaisons = get_liaisons(self.workspace, team)
            if 'email' in govteam['ptl']:
                liaisons.append(govteam['ptl']['email'])
            LOG.debug('%s needs %s' % (deliv_file, liaisons))

            for approver in self.approvers:
                if approver in liaisons:
                    print('✓ %s validated by %s' % (deliv_file, approver))
                    break
            else:
                print('✕ %s missing PTL/liaison approval' % deliv_file)
                approved = False
        return approved

def main(args=sys.argv[1:]):
    parser = argparse.ArgumentParser()
    parser.add_argument('changeid')
    parser.add_argument('releases')
    parser.add_argument('governance')
    parser.add_argument("--debug", action='store_true')
    args = parser.parse_args(args)

    if (args.debug):
        logging.basicConfig(level=logging.DEBUG)

    change = GerritChange(args)

    if not change.is_approved():
        sys.exit(1)

if __name__ == '__main__':
    main()
<commit_msg>check-release-approval: handle no-review case<commit_after>#!/usr/bin/env python3
#
# Check PTL/liaison has approved release
#
# Copyright 2019 Thierry Carrez <thierry@openstack.org>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import argparse
import json
import logging
import os
import sys

import requests
from requests.packages import urllib3
import yaml

PROJECTS_YAML = 'reference/projects.yaml'
GERRIT_URL = 'https://review.opendev.org/'
LOG = logging.getLogger(__name__)

# Turn of warnings about bad SSL config.
# https://urllib3.readthedocs.io/en/latest/advanced-usage.html#ssl-warnings
urllib3.disable_warnings()

def get_team(workspace, deliverablefile):
    with open(os.path.join(workspace, deliverablefile), 'r') as dfile:
        team = yaml.safe_load(dfile)['team']
    return team

def get_liaisons(workspace, team):
    filename = os.path.join(workspace, 'data/release_liaisons.yaml')
    with open(filename, 'r') as lfile:
        liaisons = yaml.safe_load(lfile)
    if team in liaisons:
        return [i['email'] for i in liaisons[team]]
    else:
        print('WARNING: %s team does not exist in liaisons file' % team)
        return []

class GerritChange(object):

    def __init__(self, args):
        # Load governance data
        with open(os.path.join(args.governance, PROJECTS_YAML), 'r') as dfile:
            self.gov_data = yaml.safe_load(dfile)

        self.load_from_gerrit(args.changeid)
        self.workspace = args.releases

    def load_from_gerrit(self, changeid):
        # Grab changeid details from Gerrit
        call = 'changes/%s' % changeid + \
               '?o=CURRENT_REVISION&o=CURRENT_FILES&o=DETAILED_LABELS' + \
               '&o=DETAILED_ACCOUNTS'
        raw = requests.get(GERRIT_URL + call)

        # Gerrit's REST API prepends a JSON-breaker to avoid XSS
        if raw.text.startswith(")]}'"):
            trimmed = raw.text[4:]
        else:
            trimmed = raw.text

        # Try to decode and bail with much detail if it fails
        try:
            decoded = json.loads(trimmed)
        except Exception:
            LOG.error(
                '\nrequest returned %s error to query:\n\n    %s\n'
                '\nwith detail:\n\n    %s\n',
                raw, raw.url, trimmed)
            raise

        # Extract approvers from JSON data. Approvers include change owner
        # and anyone who voted Code-Review+1. NB: Gerrit does not fill
        # labels.CodeReview.all unless there is a vote already
        self.approvers = [decoded['owner']['email']]
        if 'all' in decoded['labels']['Code-Review']:
            self.approvers.extend([
                i['email']
                for i in decoded['labels']['Code-Review']['all']
                if i['value'] > 0
            ])

        # Extract list of modified deliverables files from JSON data
        currev = decoded['current_revision']
        self.deliv_files = [
            x for x in decoded['revisions'][currev]['files'].keys()
            if x.startswith('deliverables/')
        ]

    def is_approved(self):
        LOG.debug('Approvals: %s' % self.approvers)
        approved = True
        for deliv_file in self.deliv_files:
            team = get_team(self.workspace, deliv_file)
            try:
                govteam = self.gov_data[team]
            except ValueError:
                print('✕ %s mentions unknown team %s' % (deliv_file, team))
                approved = False
                break

            # Check that deliverable is indeed defined in governance team
            delivname, _ = os.path.splitext(os.path.basename(deliv_file))
            if delivname not in govteam['deliverables']:
                print('✕ %s not in %s governance' % (deliv_file, team))
                approved = False
                break

            # Fetch PTL and release liaisons
            liaisons = get_liaisons(self.workspace, team)
            if 'email' in govteam['ptl']:
                liaisons.append(govteam['ptl']['email'])
            LOG.debug('%s needs %s' % (deliv_file, liaisons))

            for approver in self.approvers:
                if approver in liaisons:
                    print('✓ %s validated by %s' % (deliv_file, approver))
                    break
            else:
                print('✕ %s missing PTL/liaison approval' % deliv_file)
                approved = False
        return approved

def main(args=sys.argv[1:]):
    parser = argparse.ArgumentParser()
    parser.add_argument('changeid')
    parser.add_argument('releases')
    parser.add_argument('governance')
    parser.add_argument("--debug", action='store_true')
    args = parser.parse_args(args)

    if (args.debug):
        logging.basicConfig(level=logging.DEBUG)

    change = GerritChange(args)

    if not change.is_approved():
        sys.exit(1)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: speaker-recognition.py
# Date: Sun Feb 22 21:19:52 2015 +0800
# Author: Yuxin Wu <ppwwyyxxc@gmail.com>

import argparse
import sys
import glob
import os
import itertools
import scipy.io.wavfile as wavfile

sys.path.append(os.path.join(
    os.path.dirname(os.path.realpath(__file__)),
    'gui'))
from gui.interface import ModelInterface
from gui.utils import read_wav
from filters.silence import remove_silence

def get_args():
    desc = "Speaker Recognition Command Line Tool"
    epilog = """
Wav files in each input directory will be labeled as the basename of the directory.
Note that wildcard inputs should be *quoted*, and they will be sent to glob module.

Examples:
    Train:
    ./speaker-recognition.py -t enroll -i "/tmp/person* ./mary" -m model.out

    Predict:
    ./speaker-recognition.py -t predict -i "./*.wav" -m model.out
"""
    parser = argparse.ArgumentParser(description=desc,epilog=epilog,
                                    formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('-t', '--task',
                       help='Task to do. Either "enroll" or "predict"',
                       required=True)

    parser.add_argument('-i', '--input',
                       help='Input Files(to predict) or Directories(to enroll)',
                       required=True)

    parser.add_argument('-m', '--model',
                       help='Model file to save(in enroll) or use(in predict)',
                       required=True)

    ret = parser.parse_args()
    return ret

def task_enroll(input_dirs, output_model):
    m = ModelInterface()
    input_dirs = [os.path.expanduser(k) for k in input_dirs.strip().split()]
    dirs = itertools.chain(*(glob.glob(d) for d in input_dirs))
    dirs = [d for d in dirs if os.path.isdir(d)]
    files = []
    if len(dirs) == 0:
        print "No valid directory found!"
        sys.exit(1)
    for d in dirs:
        label = os.path.basename(os.path.dirname(d))

        wavs = glob.glob(d + '/*.wav')
        if len(wavs) == 0:
            print "No wav file found in {0}".format(d)
            continue
        print "Label {0} has files {1}".format(label, ','.join(wavs))
        for wav in wavs:
            fs, signal = read_wav(wav)
            m.enroll(label, fs, signal)

    m.train()
    m.dump(output_model)

def task_predict(input_files, input_model):
    m = ModelInterface.load(input_model)
    for f in glob.glob(os.path.expanduser(input_files)):
        fs, signal = read_wav(f)
        label = m.predict(fs, signal)
        print f, '->', label

if __name__ == '__main__':
    global args
    args = get_args()

    task = args.task
    if task == 'enroll':
        task_enroll(args.input, args.model)
    elif task == 'predict':
        task_predict(args.input, args.model)
<commit_msg>fix dirname<commit_after>#!/usr/bin/env python2
# -*- coding: UTF-8 -*-
# File: speaker-recognition.py
# Date: Sun Feb 22 22:36:46 2015 +0800
# Author: Yuxin Wu <ppwwyyxxc@gmail.com>

import argparse
import sys
import glob
import os
import itertools
import scipy.io.wavfile as wavfile

sys.path.append(os.path.join(
    os.path.dirname(os.path.realpath(__file__)),
    'gui'))
from gui.interface import ModelInterface
from gui.utils import read_wav
from filters.silence import remove_silence

def get_args():
    desc = "Speaker Recognition Command Line Tool"
    epilog = """
Wav files in each input directory will be labeled as the basename of the directory.
Note that wildcard inputs should be *quoted*, and they will be sent to glob module.

Examples:
    Train:
    ./speaker-recognition.py -t enroll -i "/tmp/person* ./mary" -m model.out

    Predict:
    ./speaker-recognition.py -t predict -i "./*.wav" -m model.out
"""
    parser = argparse.ArgumentParser(description=desc,epilog=epilog,
                                    formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument('-t', '--task',
                       help='Task to do. Either "enroll" or "predict"',
                       required=True)

    parser.add_argument('-i', '--input',
                       help='Input Files(to predict) or Directories(to enroll)',
                       required=True)

    parser.add_argument('-m', '--model',
                       help='Model file to save(in enroll) or use(in predict)',
                       required=True)

    ret = parser.parse_args()
    return ret

def task_enroll(input_dirs, output_model):
    m = ModelInterface()
    input_dirs = [os.path.expanduser(k) for k in input_dirs.strip().split()]
    dirs = itertools.chain(*(glob.glob(d) for d in input_dirs))
    dirs = [d for d in dirs if os.path.isdir(d)]
    files = []
    if len(dirs) == 0:
        print "No valid directory found!"
        sys.exit(1)
    for d in dirs:
        label = os.path.basename(d.rstrip('/'))

        wavs = glob.glob(d + '/*.wav')
        if len(wavs) == 0:
            print "No wav file found in {0}".format(d)
            continue
        print "Label {0} has files {1}".format(label, ','.join(wavs))
        for wav in wavs:
            fs, signal = read_wav(wav)
            m.enroll(label, fs, signal)

    m.train()
    m.dump(output_model)

def task_predict(input_files, input_model):
    m = ModelInterface.load(input_model)
    for f in glob.glob(os.path.expanduser(input_files)):
        fs, signal = read_wav(f)
        label = m.predict(fs, signal)
        print f, '->', label

if __name__ == '__main__':
    global args
    args = get_args()

    task = args.task
    if task == 'enroll':
        task_enroll(args.input, args.model)
    elif task == 'predict':
        task_predict(args.input, args.model)
<|endoftext|>


<commit_before># ---------------------------------------------------------------------------------------------------------------------
#
# script to import each GNAF & administrative boundary table from Postgres & export as GZIPped Parquet files
#
# PROCESS
#
# 1. get list of tables from Postgres
# 2. for each table:
#     a. import into GeoPandas dataframe
#     b. export as gzip geoparquet files to local disk using pyarrow
#
# ---------------------------------------------------------------------------------------------------------------------

import argparse
import geopandas
import json
import logging
import math
import os
import psycopg2
import pyarrow as pa
import pyarrow.parquet as pq
import pyproj
import sqlalchemy
import sys

# from boto3.s3.transfer import TransferConfig  # S3 transfer disabled as AWS CLI sync is much faster
from datetime import datetime
from multiprocessing import cpu_count
from pathlib import Path

# setup logging - code is here to prevent conflict with logging.basicConfig() from one of the imports below
log_file = os.path.abspath(__file__).replace(".py", ".log")
logging.basicConfig(filename=log_file, level=logging.DEBUG, format="%(asctime)s %(message)s",
                    datefmt="%m/%d/%Y %I:%M:%S %p")

# set number of parallel processes (sets number of Spark executors and concurrent Postgres JDBC connections)
num_processors = cpu_count()

# set postgres connection parameters
def get_password(connection_name):
    # get credentials from local file
    passwords_file_path = os.path.join(os.environ["GIT_HOME"], "passwords.ini")

    if os.path.exists(passwords_file_path):
        passwords_file = open(passwords_file_path,'r').read().splitlines()
        passwords_file = [i for i in passwords_file if len(i) != 0]  # remove empty lines
        passwords_file = [i for i in passwords_file if i[0] != "#"]  # remove comment lines

        params = dict()
        for ini in passwords_file:
            params[ini.split()[0].rstrip().lstrip()] = ini.split(':=')[1].rstrip().lstrip()

        return dict(item.split("|") for item in params[connection_name].split(","))

pg_settings = get_password("localhost_super")

# # create Postgres JDBC url
# jdbc_url = "jdbc:postgresql://{HOST}:{PORT}/{DB}".format(**pg_settings)

# get connect string for psycopg2
pg_connect_string = "dbname={DB} host={HOST} port={PORT} user={USER} password={PASS}".format(**pg_settings)

# get connect string for sqlalchemy
sql_alchemy_engine_string = "postgresql+psycopg2://{USER}:{PASS}@{HOST}:{PORT}/{DB}".format(**pg_settings)

# Set PyGEOS to True to speed up GeoPandas
geopandas.options.use_pygeos = True

# get runtime arguments
parser = argparse.ArgumentParser(description="Converts Postgres/PostGIS tables to Geoparquet files.")

parser.add_argument("--gnaf-schema", help="Input schema name for GNAF tables.")
parser.add_argument("--admin-schema", help="Input schema name for admin boundary tables.")
parser.add_argument("--output-path", help="Output path for Geoparquet files.")

def main():
    start_time = datetime.now()

    # get settings
    args = parser.parse_args()
    output_path = args.output_path
    schema_names = [args.gnaf_schema, args.admin_schema]

    # create output path (if required)
    Path(output_path).mkdir(parents=True, exist_ok=True)

    # create sqlalchemy database engine
    sql_engine = sqlalchemy.create_engine(sql_alchemy_engine_string, isolation_level="AUTOCOMMIT")

    # get list of tables to export to S3
    pg_conn = psycopg2.connect(pg_connect_string)
    pg_cur = pg_conn.cursor()

    # --------------------------------------------------------------
    # import each table from each schema in Postgres &
    # export to GZIPped Parquet files in AWS S3
    # --------------------------------------------------------------

    for schema_name in schema_names:
        i = 1

        # get table list for schema
        sql = f"""SELECT table_name
                  FROM information_schema.tables
                  WHERE table_schema='{schema_name}'
                    AND table_type='BASE TABLE'
                    AND table_name <> 'qa'
                    AND table_name NOT LIKE '%_2011_%'
                    AND table_name NOT LIKE '%_analysis%'
                    AND table_name NOT LIKE '%_display%'"""
        pg_cur.execute(sql)

        tables = pg_cur.fetchall()

        logger.info("\t - {} schema : {} tables to export : {}"
                    .format(schema_name, len(tables), datetime.now() - start_time))

        for table in tables:
            start_time = datetime.now()

            table_name = table[0]

            # check what type of geometry field the table has and what it's coordinate system is
            sql = f"""SELECT f_geometry_column, type, srid FROM public.geometry_columns
                      WHERE f_table_schema = '{schema_name}'
                          AND f_table_name = '{table_name}'"""
            pg_cur.execute(sql)
            result = pg_cur.fetchone()

            if result is not None:
                geom_field = result[0]
                geom_type = result[1]
                geom_srid = result[2]
            else:
                geom_field = None
                geom_type = None
                geom_srid = None

            # build geom field sql
            # note: exported geom field will be WGS84 (EPSG:4326) Well Known Binaries (WKB)
            if geom_field is not None:
                if "POLYGON" in geom_type or "LINESTRING" in geom_type:
                    if geom_srid != 4326:
                        geom_sql = ",ST_Subdivide((ST_Dump(ST_Buffer(ST_Transform(geom, 4326), 0.0))).geom, 256) " \
                                   "as geometry"
                    else:
                        geom_sql = ",ST_Subdivide((ST_Dump(ST_Buffer(geom, 0.0))).geom, 256) as geometry"
                else:
                    if geom_srid != 4326:
                        geom_sql = ",ST_Transform(geom, 4326) as geometry"

                    else:
                        geom_sql = ",geom as geometry"

            else:
                geom_sql = ""

            # build query to select all columns and the WKB geom if exists
            sql = f"""SELECT 'SELECT ' || array_to_string(ARRAY(
                          SELECT column_name
                          FROM information_schema.columns
                          WHERE table_name = '{table_name}'
                              AND table_schema = '{schema_name}'
                              AND column_name NOT IN('geom')
                      ), ',') || '{geom_sql} ' ||
                      'FROM {schema_name}.{table_name}' AS sqlstmt"""
            pg_cur.execute(sql)

            import_query = str(pg_cur.fetchone()[0])  # str is just there for intellisense in Pycharm

            # import into GeoPandas
            df = import_table(sql_engine, import_query)
            # num_rows = df.shape[0]
            # logger.info(f"\t\t {i}. imported {num_rows} from {table_name} : {datetime.now() - start_time}")
            # start_time = datetime.now()

            # export
            export_to_geoparquet(df, geom_type, table_name, output_path)

            logger.info(f"\t\t {i}. exported {table_name} : {datetime.now() - start_time}")
            # else:
            #     logger.warning("\t\t {}. {} has no records! : {}".format(i, table_name, datetime.now() - start_time))

            i += 1

    # cleanup
    pg_cur.close()
    pg_conn.close()

# load bdy table from Postgres and create a geospatial dataframe from it
def import_table(sql_engine, sql):

    # debugging
    sql += " where state = 'ACT'"

    df = geopandas.GeoDataFrame.from_postgis(sql, sql_engine, geom_col='geometry')
    return df

# export a dataframe to gz parquet files
def export_to_geoparquet(df, geom_type, name, output_path):

    table = pa.Table.from_pandas(df.to_wkb())

    # add metadata & schema
    metadata = {
        "version": "0.3.0",
        "primary_column": "geometry",
        "columns": {
            "geometry": {
                "encoding": "WKB",
                "geometry_type": [geom_type.capitalize()],
                "crs": df.crs.to_wkt(pyproj.enums.WktVersion.WKT2_2019_SIMPLIFIED),
                "edges": "planar",
                "bbox": [round(x, 4) for x in df.total_bounds],
            },
        },
    }

    schema = (
        table.schema
            .with_metadata({"geo": json.dumps(metadata)})
    )
    table = table.cast(schema)

    # export to geoparquet
    pq.write_table(table, os.path.join(output_path, f"{name}.parquet"), compression="snappy")

    # df.write.option("compression", "gzip") \
    #     .mode("overwrite") \
    #     .parquet(os.path.join(output_path, name))

# def copy_to_s3(schema_name, name, output_path):
#
#     # set correct AWS user
#     boto3.setup_default_session(profile_name="minus34")
#
#     # delete existing files (each time you run this Spark creates new, random parquet file names)
#     s3 = boto3.resource('s3')
#     bucket = s3.Bucket(s3_bucket)
#     bucket.objects.filter(Prefix=os.path.join(s3_folder, schema_name, name)).delete()
#
#     s3_client = boto3.client('s3')
#     config = TransferConfig(multipart_threshold=1024 ** 2)  # 1MB
#
#     # upload one file at a time
#     for root, dirs, files in os.walk(os.path.join(output_path, name)):
#         for file in files:
#             response = s3_client\
#                 .upload_file(os.path.join(output_path, name, file), s3_bucket,
#                              os.path.join(s3_folder, schema_name, name, file),
#                              Config=config, ExtraArgs={'ACL': 'public-read'})
#
#             if response is not None:
#                 logger.warning("\t\t\t - {} copy to S3 problem : {}".format(name, response))

if __name__ == "__main__":
    full_start_time = datetime.now()

    # setup logging to file and the console (screen)
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)

    # set Spark logging levels
    logging.getLogger("pyspark").setLevel(logging.ERROR)
    logging.getLogger("py4j").setLevel(logging.ERROR)

    # setup logger to write to screen as well as writing to log file
    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    # set a format which is simpler for console use
    formatter = logging.Formatter("%(name)-12s: %(levelname)-8s %(message)s")
    # tell the handler to use this format
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger().addHandler(console)

    task_name = "Geoscape Admin Boundary Export to S3"

    logger.info("{} started".format(task_name))
    logger.info("Running on Python {}".format(sys.version.replace("\n", " ")))

    main()

    time_taken = datetime.now() - full_start_time
    logger.info("{} finished : {}".format(task_name, time_taken))
    print()
<commit_msg>testing<commit_after># ---------------------------------------------------------------------------------------------------------------------
#
# script to import each GNAF & administrative boundary table from Postgres & export as GZIPped Parquet files
#
# PROCESS
#
# 1. get list of tables from Postgres
# 2. for each table:
#     a. import into GeoPandas dataframe
#     b. export as gzip geoparquet files to local disk using pyarrow
#
# ---------------------------------------------------------------------------------------------------------------------

import argparse
import geopandas
import json
import logging
import math
import os
import psycopg2
import pyarrow as pa
import pyarrow.parquet as pq
import pyproj
import sqlalchemy
import sys

# from boto3.s3.transfer import TransferConfig  # S3 transfer disabled as AWS CLI sync is much faster
from datetime import datetime
from multiprocessing import cpu_count
from pathlib import Path

# setup logging - code is here to prevent conflict with logging.basicConfig() from one of the imports below
log_file = os.path.abspath(__file__).replace(".py", ".log")
logging.basicConfig(filename=log_file, level=logging.DEBUG, format="%(asctime)s %(message)s",
                    datefmt="%m/%d/%Y %I:%M:%S %p")

# set number of parallel processes (sets number of Spark executors and concurrent Postgres JDBC connections)
num_processors = cpu_count()

# set postgres connection parameters
def get_password(connection_name):
    # get credentials from local file
    passwords_file_path = os.path.join(os.environ["GIT_HOME"], "passwords.ini")

    if os.path.exists(passwords_file_path):
        passwords_file = open(passwords_file_path,'r').read().splitlines()
        passwords_file = [i for i in passwords_file if len(i) != 0]  # remove empty lines
        passwords_file = [i for i in passwords_file if i[0] != "#"]  # remove comment lines

        params = dict()
        for ini in passwords_file:
            params[ini.split()[0].rstrip().lstrip()] = ini.split(':=')[1].rstrip().lstrip()

        return dict(item.split("|") for item in params[connection_name].split(","))

pg_settings = get_password("localhost_super")

# # create Postgres JDBC url
# jdbc_url = "jdbc:postgresql://{HOST}:{PORT}/{DB}".format(**pg_settings)

# get connect string for psycopg2
pg_connect_string = "dbname={DB} host={HOST} port={PORT} user={USER} password={PASS}".format(**pg_settings)

# get connect string for sqlalchemy
sql_alchemy_engine_string = "postgresql+psycopg2://{USER}:{PASS}@{HOST}:{PORT}/{DB}".format(**pg_settings)

# Set PyGEOS to True to speed up GeoPandas
geopandas.options.use_pygeos = True

# get runtime arguments
parser = argparse.ArgumentParser(description="Converts Postgres/PostGIS tables to Geoparquet files.")

parser.add_argument("--gnaf-schema", help="Input schema name for GNAF tables.")
parser.add_argument("--admin-schema", help="Input schema name for admin boundary tables.")
parser.add_argument("--output-path", help="Output path for Geoparquet files.")

def main():
    start_time = datetime.now()

    # get settings
    args = parser.parse_args()
    output_path = args.output_path
    schema_names = [args.gnaf_schema, args.admin_schema]

    # create output path (if required)
    Path(output_path).mkdir(parents=True, exist_ok=True)

    # create sqlalchemy database engine
    sql_engine = sqlalchemy.create_engine(sql_alchemy_engine_string, isolation_level="AUTOCOMMIT")

    # get list of tables to export to S3
    pg_conn = psycopg2.connect(pg_connect_string)
    pg_cur = pg_conn.cursor()

    # --------------------------------------------------------------
    # import each table from each schema in Postgres &
    # export to GZIPped Parquet files in AWS S3
    # --------------------------------------------------------------

    for schema_name in schema_names:
        i = 1

        # get table list for schema
        sql = f"""SELECT table_name
                  FROM information_schema.tables
                  WHERE table_schema='{schema_name}'
                    AND table_type='BASE TABLE'
                    AND table_name <> 'qa'
                    AND table_name NOT LIKE '%_2011_%'
                    AND table_name NOT LIKE '%_analysis%'
                    AND table_name NOT LIKE '%_display%'"""
        pg_cur.execute(sql)

        tables = pg_cur.fetchall()

        logger.info("\t - {} schema : {} tables to export : {}"
                    .format(schema_name, len(tables), datetime.now() - start_time))

        for table in tables:
            start_time = datetime.now()

            table_name = table[0]

            # check what type of geometry field the table has and what it's coordinate system is
            sql = f"""SELECT f_geometry_column, type, srid FROM public.geometry_columns
                      WHERE f_table_schema = '{schema_name}'
                          AND f_table_name = '{table_name}'"""
            pg_cur.execute(sql)
            result = pg_cur.fetchone()

            if result is not None:
                geom_field = result[0]
                geom_type = result[1]
                geom_srid = result[2]
            else:
                geom_field = None
                geom_type = None
                geom_srid = None

            # build geom field sql
            # note: exported geom field will be WGS84 (EPSG:4326) Well Known Binaries (WKB)
            if geom_field is not None:
                if "POLYGON" in geom_type or "LINESTRING" in geom_type:
                    if geom_srid != 4326:
                        geom_sql = ",ST_Subdivide((ST_Dump(ST_Buffer(ST_Transform(geom, 4326), 0.0))).geom, 256) " \
                                   "as geometry"
                    else:
                        geom_sql = ",ST_Subdivide((ST_Dump(ST_Buffer(geom, 0.0))).geom, 256) as geometry"
                else:
                    if geom_srid != 4326:
                        geom_sql = ",ST_Transform(geom, 4326) as geometry"

                    else:
                        geom_sql = ",geom as geometry"

            else:
                geom_sql = ""

            # build query to select all columns and the WKB geom if exists
            sql = f"""SELECT 'SELECT ' || array_to_string(ARRAY(
                          SELECT column_name
                          FROM information_schema.columns
                          WHERE table_name = '{table_name}'
                              AND table_schema = '{schema_name}'
                              AND column_name NOT IN('geom')
                      ), ',') || '{geom_sql} ' ||
                      'FROM {schema_name}.{table_name}' AS sqlstmt"""
            pg_cur.execute(sql)

            import_query = str(pg_cur.fetchone()[0])  # str is just there for intellisense in Pycharm

            # import into GeoPandas
            df = import_table(sql_engine, import_query)
            # num_rows = df.shape[0]
            # logger.info(f"\t\t {i}. imported {num_rows} from {table_name} : {datetime.now() - start_time}")
            # start_time = datetime.now()

            # export
            export_to_geoparquet(df, geom_type, table_name, output_path)

            logger.info(f"\t\t {i}. exported {table_name} : {datetime.now() - start_time}")
            # else:
            #     logger.warning("\t\t {}. {} has no records! : {}".format(i, table_name, datetime.now() - start_time))

            i += 1

    # cleanup
    pg_cur.close()
    pg_conn.close()

# load bdy table from Postgres and create a geospatial dataframe from it
def import_table(sql_engine, sql):
    # # debugging
    # sql += " where state = 'ACT'"

    df = geopandas.GeoDataFrame.from_postgis(sql, sql_engine, geom_col='geometry')
    return df

# export a dataframe to gz parquet files
def export_to_geoparquet(df, geom_type, name, output_path):

    table = pa.Table.from_pandas(df.to_wkb())

    # add metadata & schema
    metadata = {
        "version": "0.3.0",
        "primary_column": "geometry",
        "columns": {
            "geometry": {
                "encoding": "WKB",
                "geometry_type": [geom_type.capitalize()],
                "crs": df.crs.to_wkt(pyproj.enums.WktVersion.WKT2_2019_SIMPLIFIED),
                "edges": "planar",
                "bbox": [round(x, 4) for x in df.total_bounds],
            },
        },
    }

    schema = (
        table.schema
            .with_metadata({"geo": json.dumps(metadata)})
    )
    table = table.cast(schema)

    # export to geoparquet
    pq.write_table(table, os.path.join(output_path, f"{name}.parquet"), compression="snappy")

    # df.write.option("compression", "gzip") \
    #     .mode("overwrite") \
    #     .parquet(os.path.join(output_path, name))

# def copy_to_s3(schema_name, name, output_path):
#
#     # set correct AWS user
#     boto3.setup_default_session(profile_name="minus34")
#
#     # delete existing files (each time you run this Spark creates new, random parquet file names)
#     s3 = boto3.resource('s3')
#     bucket = s3.Bucket(s3_bucket)
#     bucket.objects.filter(Prefix=os.path.join(s3_folder, schema_name, name)).delete()
#
#     s3_client = boto3.client('s3')
#     config = TransferConfig(multipart_threshold=1024 ** 2)  # 1MB
#
#     # upload one file at a time
#     for root, dirs, files in os.walk(os.path.join(output_path, name)):
#         for file in files:
#             response = s3_client\
#                 .upload_file(os.path.join(output_path, name, file), s3_bucket,
#                              os.path.join(s3_folder, schema_name, name, file),
#                              Config=config, ExtraArgs={'ACL': 'public-read'})
#
#             if response is not None:
#                 logger.warning("\t\t\t - {} copy to S3 problem : {}".format(name, response))

if __name__ == "__main__":
    full_start_time = datetime.now()

    # setup logging to file and the console (screen)
    logger = logging.getLogger()
    logger.setLevel(logging.INFO)

    # set Spark logging levels
    logging.getLogger("pyspark").setLevel(logging.ERROR)
    logging.getLogger("py4j").setLevel(logging.ERROR)

    # setup logger to write to screen as well as writing to log file
    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    # set a format which is simpler for console use
    formatter = logging.Formatter("%(name)-12s: %(levelname)-8s %(message)s")
    # tell the handler to use this format
    console.setFormatter(formatter)
    # add the handler to the root logger
    logging.getLogger().addHandler(console)

    task_name = "Geoscape Admin Boundary Export to S3"

    logger.info("{} started".format(task_name))
    logger.info("Running on Python {}".format(sys.version.replace("\n", " ")))

    main()

    time_taken = datetime.now() - full_start_time
    logger.info("{} finished : {}".format(task_name, time_taken))
    print()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# This file is execfile()d with the current directory set to its containing dir.

from textwrap import dedent
import os
import sys

import jsonschema

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
ext_paths = [os.path.abspath(os.path.pardir), os.path.dirname(__file__)]
sys.path = ext_paths + sys.path

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = "1.0"

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named "sphinx.ext.*") or your custom ones.
extensions = [
    "sphinx.ext.autodoc",
    "sphinx.ext.coverage",
    "sphinx.ext.doctest",
    "sphinx.ext.intersphinx",
    "sphinx.ext.napoleon",
    "sphinx.ext.viewcode",
    "sphinxcontrib.spelling",
    "jsonschema_role",
]

cache_path = "_cache"

# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

# The suffix of source filenames.
source_suffix = ".rst"

# The encoding of source files.
# source_encoding = "utf-8-sig"

# The master toctree document.
master_doc = "index"

# General information about the project.
project = u"jsonschema"
author = u"Julian Berman"
copyright = u"2013, " + author

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# version: The short X.Y version
# release: The full version, including alpha/beta/rc tags.
release = jsonschema.__version__
version = release.partition("-")[0]

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ""
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = "%B %d, %Y"

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ["_build", "_cache", "_static", "_templates"]

# The reST default role (used for this markup: `text`) to use for all documents.
# default_role = None

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

doctest_global_setup = dedent("""
    from __future__ import print_function
    from jsonschema import *
""")

intersphinx_mapping = {
    "python": ("https://docs.python.org/2.7", None),
    "python3": ("https://docs.python.org/3", None),
}

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = "pyramid"

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ["_static"]

# If not "", a "Last updated on:" timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = "%b %d, %Y"

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ""

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = "jsonschemadoc"

# -- Options for LaTeX output --------------------------------------------------

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  (
      "index",
      "jsonschema.tex",
      u"jsonschema Documentation",
      author,
      "manual",
  ),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (
        "index",
        "jsonschema",
        u"jsonschema Documentation",
        [author],
        1,
    ),
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (
      "index",
      "jsonschema",
      u"jsonschema Documentation",
      author,
      "jsonschema",
      "One line description of project.",
      "Miscellaneous",
  ),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: "footnote", "no", or "inline".
# texinfo_show_urls = "footnote"

# -- Options for sphinxcontrib-spelling -----------------------------------

spelling_word_list_filename = "spelling-wordlist.txt"
<commit_msg>Start taming reST.<commit_after># -*- coding: utf-8 -*-
#
# This file is execfile()d with the current directory set to its containing dir.

from textwrap import dedent
import os
import sys

import jsonschema

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
ext_paths = [os.path.abspath(os.path.pardir), os.path.dirname(__file__)]
sys.path = ext_paths + sys.path

# -- General configuration -----------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = "1.0"

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named "sphinx.ext.*") or your custom ones.
extensions = [
    "sphinx.ext.autodoc",
    "sphinx.ext.coverage",
    "sphinx.ext.doctest",
    "sphinx.ext.intersphinx",
    "sphinx.ext.napoleon",
    "sphinx.ext.viewcode",
    "sphinxcontrib.spelling",
    "jsonschema_role",
]

cache_path = "_cache"

# Add any paths that contain templates here, relative to this directory.
templates_path = ["_templates"]

# The suffix of source filenames.
source_suffix = ".rst"

# The encoding of source files.
# source_encoding = "utf-8-sig"

# The master toctree document.
master_doc = "index"

# General information about the project.
project = u"jsonschema"
author = u"Julian Berman"
copyright = u"2013, " + author

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# version: The short X.Y version
# release: The full version, including alpha/beta/rc tags.
release = jsonschema.__version__
version = release.partition("-")[0]

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ""
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = "%B %d, %Y"

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ["_build", "_cache", "_static", "_templates"]

# The reST default role (used for this markup: `text`) to use for all documents.
default_role = "any"

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = "sphinx"

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

doctest_global_setup = dedent("""
    from __future__ import print_function
    from jsonschema import *
""")

intersphinx_mapping = {
    "python": ("https://docs.python.org/2.7", None),
    "python3": ("https://docs.python.org/3", None),
}

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = "pyramid"

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ["_static"]

# If not "", a "Last updated on:" timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = "%b %d, %Y"

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ""

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = "jsonschemadoc"

# -- Options for LaTeX output --------------------------------------------------

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  (
      "index",
      "jsonschema.tex",
      u"jsonschema Documentation",
      author,
      "manual",
  ),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output --------------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (
        "index",
        "jsonschema",
        u"jsonschema Documentation",
        [author],
        1,
    ),
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output ------------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (
      "index",
      "jsonschema",
      u"jsonschema Documentation",
      author,
      "jsonschema",
      "One line description of project.",
      "Miscellaneous",
  ),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: "footnote", "no", or "inline".
# texinfo_show_urls = "footnote"

# -- Options for sphinxcontrib-spelling -----------------------------------

spelling_word_list_filename = "spelling-wordlist.txt"
<|endoftext|>