<commit_before>#!/usr/bin/python3

import errno
import os
import re
import shutil
from subprocess import Popen, PIPE
from stawebg.config import Config
from stawebg.helper import (listFolders, findFiles, findDirs, fail, matchList,
                            cleverCapitalize)

version = "0.1"

isFile = lambda f: os.path.isfile(f)
matchPath = lambda f, c: os.path.abspath(f)[len(os.path.abspath(c.getAbsSrcPath()))+1:]
isIndex = lambda f, site: matchList(matchPath(f, site), site.getConfig(['files', 'index']))
isCont = lambda f, site: matchList(matchPath(f, site), site.getConfig(['files', 'content']))
isExcluded = lambda f, site, c: matchList(matchPath(f, site), c.get(['files', 'exclude'], False, []))
isHidden = lambda f, site, c: matchList(matchPath(f, site), c.get(['files', 'hidden'], False, []))

class Project:
    def __init__(self, project_dir="", test=False):
        self._sites = []
        self._layouts = {}
        self._root_dir = project_dir
        self._test = test

        self._config = Config(os.path.join(self._root_dir, "stawebg.json"),
                              Config.global_struct)

        # Make directories absolute
        dirs = self._config.get(["dirs"])
        for k in dirs:
            dirs[k] = os.path.join(self._root_dir, dirs[k])

    def read(self):
        self._readLayouts()
        self._readSites()

    def copy(self):
        for s in self._sites:
            s.copy()

    def getConfig(self, key, fail=True, default=None):
        return self._config.get(key, fail, default)

    def getLayout(self, name=None):
        if not name:
            name = "default"

        layout = self._layouts.get(name)

        if not layout:
            fail("Can't find layout: " + name)

        return layout

    def getOutputDir(self):
        if self._test:
            return self.getConfig(["dirs", "test"])
        else:
            return self.getConfig(["dirs", "out"])

    # Add all layouts to list
    def _readLayouts(self):
        for name in listFolders(self.getConfig(['dirs', 'layouts'])):
            self._layouts[name] = Layout(self, name)
            self._layouts[name].read()

    # Add all site directories to list
    def _readSites(self):
        for s in listFolders(self.getConfig(['dirs', 'sites'])):
            site = Site(s, self)
            self._sites.append(site)
            site.read()

class Layout:
    def __init__(self, project, name):
        self._project = project
        self._name = name
        self._dir = os.path.join(self._project.getConfig(['dirs', 'layouts']),
                                 name)
        self._template = os.path.join(self._dir, 'template.html')
        self._other_files = []

        print("Found layout: " + self._name)

    def read(self):
        for f in findFiles(self._dir, [".html"]):
            print("\tFound file: " + f)
            self._other_files.append(OtherFile(self._dir,
                                               os.path.relpath(f, self._dir)))

    def copy(self, dest, site):
        for f in self._other_files:
            f.copy(site, os.path.join(dest, self.getSubdir()))

    def getSubdir(self):
        return os.path.join("style", self._name)

    def getTemplate(self):
        return self._template

class Site:
    def __init__(self, name, project):
        self._project = project
        self._name = name
        self._root = None
        self._other_files = []
        self._config = self._project._config
        self._layouts = []
        self._file_index = []
        print("Found site: " + self._name)

    def getConfig(self, key, fail=True, default=None):
        return self._config.get(key, fail, default)

    def getAbsSrcPath(self):
        return os.path.join(self.getConfig(['dirs', "sites"]), self._name)

    def getAbsDestPath(self):
        return os.path.join(self._project.getOutputDir(), self._name)

    def getProject(self):
        return self._project

    def addLayout(self, name):
        layout = self._project.getLayout(name)
        if layout not in self._layouts:
            self._layouts.append(layout)

    def getSiteTitle(self):
        tmp = self.getConfig(["title"], False)
        if tmp:
            return tmp

        return self._name

    def getSiteSubtitle(self):
        tmp = self.getConfig(["subtitle"], False)
        if tmp:
            return tmp

        return ""

    def read(self):
        self._readConfig()
        self._createFileIndex()
        self._readHelper(self.getAbsSrcPath(), self._root)

    def copy(self):
        print("Create site: " + self._name)

        # Pages
        self._root.copy()

        # Layouts
        for l in self._layouts:
            l.copy(self.getAbsDestPath(), self)

        # Other files
        for f in self._other_files:
            f.copy(self)

        # Cleanup
        self._cleanupOutput()

    def _readConfig(self):
        filename = os.path.join(self.getConfig(["dirs", "sites"]),
                                self._name + ".json")

        if not os.path.isfile(filename):
            fail("Can't find config file: " + filename)

        site_config = Config(filename, Config.site_struct)
        self._config = Config.merge(self._config, site_config, True)

    def _readHelper(self, dir_path, parent, dir_hidden=False, page_config=None):
        index_rename = None
        if page_config:
            index_rename = page_config.get(["files", "rename",
                                            os.path.basename(dir_path)], False)
            page_config.delete(["files", "sort"], False)
            page_config.delete(["files", "rename"], False)
        else:
            page_config = self._config

        entries = sorted(os.listdir(dir_path))

        if "stawebg.json" in entries:
            tmp_config = Config(os.path.join(dir_path, "stawebg.json"),
                                Config.directory_struct)
            page_config = Config.merge(page_config, tmp_config, False)

        # Add layout to list -> copy later
        self.addLayout(page_config.get(["layout"], False))

        # First we have to find the index file in this directory…
        idx = None
        for f in entries:
            absf = os.path.join(dir_path, f)
            if isFile(absf) and isCont(absf, self) and isIndex(absf, self):
                if index_rename:
                    page_config.add(["files", "rename", f], index_rename)
                idx = Page(os.path.split(dir_path)[1], absf, self, parent,
                           dir_hidden or isHidden(absf, self, page_config),
                           page_config)
                entries.remove(f)
                break
        # …or create an empty page as index
        if not idx:
            dirname = os.path.split(dir_path)[1]
            if index_rename:
                page_config.add(["files", "rename", None], index_rename)
            idx = Page(dirname, None, self, parent, dir_hidden or
                       isHidden(dirname, self, page_config), page_config)

        if parent:
            parent.appendPage(idx)
        else:
            self._root = idx

        # Sort entries as specified in configuration
        sorted_entries = page_config.get(["files", "sort"], False, [])
        for s in reversed(sorted_entries):
            absf = os.path.join(dir_path, s)
            if not s in entries:
                print("\tFile not found (specified in sort): " + absf)
            else:
                entries.remove(s)
                entries.insert(0, s)

        # Make absolute paths and check if it's a page
        for f in entries:
            absf = os.path.join(dir_path, f)
            hidden = dir_hidden or isHidden(absf, self, page_config)

            # Content file -> Page
            if isFile(absf) and isCont(absf, self):
                print("\tFound page: " + absf)
                idx.appendPage(Page(os.path.splitext(f)[0], absf, self, idx,
                                    hidden, page_config))
            # Directory -> Go inside
            elif os.path.isdir(absf):
                self._readHelper(absf, idx, hidden, page_config.copy())
            # Unknown object
            else:
                if not isExcluded(absf, self, page_config):
                    tmp = OtherFile(self.getAbsSrcPath(),
                                    os.path.relpath(
                                        absf, self.getAbsSrcPath()),
                                    self.getAbsDestPath())
                    self._other_files.append(tmp)
                    print("\tFound unkown object: " + absf)

    def createMenu(self, cur_page):
        return self._root.createMenu(cur_page)

    def _createFileIndex(self):
        path = self.getAbsDestPath()
        if os.path.isdir(path):
            self._file_index = findFiles(path)

    def delFromFileIndex(self, path):
        if path in self._file_index:
            self._file_index.remove(path)

    def _rmFileIndex(self):
        for f in self._file_index:
            print("\tRemove old file: " + f)
            try:
                os.remove(f)
            except OSError as e:
                print("\tError: " + str(e))

    def _cleanupOutput(self):
        self._rmFileIndex()

        # Delete empty directories
        for d in findDirs(self.getAbsDestPath()):
            if not os.listdir(d):
                print("\tRemove empty directory: " + d)
                try:
                    os.rmdir(d)
                except OSError as e:
                    print("\tError: " + str(e))

class Page:
    def __init__(self, name, absPath, site, parent, hidden, config):
        self._name = name
        self._absSrc = absPath
        self._site = site
        self._hidden = hidden
        self._parent = parent
        self._subpages = []
        self._config = config

        self._site.delFromFileIndex(self._getDestFile())

    def appendPage(self, p):
        self._subpages.append(p)

    def getPages(self):
        return self._subpages

    def getParent(self):
        return self._parent

    def getName(self):
        return self._name

    def isRoot(self):
        return self._parent is None

    def isHidden(self):
        return self._hidden

    def getLayout(self):
        layout = self._config.get(["layout"], False)
        return self._site.getProject().getLayout(layout)

    def copy(self):
        # Use 'codecs' package to support UTF-8?
        try:
            os.makedirs(os.path.dirname(self._getDestFile()))
            outf = open(self._getDestFile(), 'w')
            tplf = open(self.getLayout().getTemplate())
            outf.write(self._replaceKeywords(tplf.read()))
            tplf.close()
            outf.close()
        # TODO: check for other possible exceptions
        except IOError as e:
            fail("Error creating " + self._getDestFile() + ": " + str(e))
        except OSError as e:
            if e.errno != errno.EEXIST:
                fail(str(e))

        # Copy subpages
        for p in self._subpages:
            p.copy()

    def _translateMarkup(self):
        text = ''

        if not self._absSrc:
            return text

        with open(self._absSrc, "rt") as src:
            text = src.read()

        config = self._site.getConfig(["markup"], False)
        if not config:
            return text
        tool = config.get(os.path.splitext(self._absSrc)[1])

        if tool:
            pipe = Popen(tool, stdin=PIPE, stdout=PIPE, stderr=PIPE)
            out, err = pipe.communicate(text.encode())
            if len(err):
                fail(' '.join(tool) + ": " + err.decode())
            return out.decode()
        return text

    def _replaceKeywords(self, text):
        reps = {"%ROOT%": self.getRootLink(),
                "%CUR%": self.getCurrentLink(),
                "%LAYOUT%": self.getLayoutDir(),
                "%TITLE%": self.getTitle(),
                "%SITETITLE%": self._site.getSiteTitle(),
                "%SITESUBTITLE%": self._site.getSiteSubtitle(),
                "%MENU%": self._site.createMenu(self)}
        trans = lambda m: reps[m.group(0)]
        rc = re.compile('|'.join(map(re.escape, reps)))
        return rc.sub(trans, text.replace("%CONTENT%", self._translateMarkup()))

    def _getDestFile(self):
        tmp_path = ""

        parent = self.getParent()
        while parent and not parent.isRoot():
            tmp_path = os.path.join(parent.getName(), tmp_path)
            parent = parent.getParent()

        if not self.isRoot():
            tmp_path = os.path.join(tmp_path, self.getName())

        return os.path.join(self._site.getAbsDestPath(), tmp_path,
                            "index.html")

    def getRootLink(self):
        tmp = ""

        parent = self.getParent()
        while parent:
            tmp = tmp + "../"
            parent = parent.getParent()

        return tmp

    def getCurrentLink(self):
        return '' if self._absSrc and isIndex(self._absSrc, self._site) else '../'

    def getLayoutDir(self):
        tmp = self.getRootLink()
        if len(tmp):
            tmp += "/"
        tmp += self.getLayout().getSubdir() + "/"
        return tmp

    def getLink(self, origin=None):
        tmp = ""
        parent = self
        while parent and not parent.isRoot():
            tmp = parent.getName() + "/" + tmp
            parent = parent.getParent()

        if not origin:
            origin = self

        tmp = origin.getRootLink() + tmp

        if tmp == "":
            tmp = "."

        return tmp

    def getShortTitle(self):
        key = None
        if self._absSrc:
            key = os.path.basename(self._absSrc)

        rename = self._config.get(["files", "rename", key], False)

        if rename:
            return rename
        elif not self.getParent():
            return "Home"
        else:
            return cleverCapitalize(self.getName())

    def getTitle(self, no_home=False):
        if not self.getParent():
            if no_home:
                return self._site.getSiteTitle()
            else:
                return self._site.getSiteTitle() + " > " + self.getShortTitle()
        else:
            return self.getParent().getTitle(True) + " > " + self.getShortTitle()

    def createMenu(self, cur_page, last=False):
        items = self._subpages[:]

        # Add root link
        if self.isRoot():
            items.insert(0, self)

        # Create HTML Code
        found = False
        tmp = "
\n"
        for p in items:
            if p == cur_page:
                found = True

            if p.isHidden():
                continue

            active = ""
            if p.pageIsInPathTo(cur_page) and (not p.isRoot() or p == cur_page):
                active = " class=\"active\""

            tmp = ''.join([tmp, "
", p.getShortTitle(), "
\n"])

            # Create submenu
            if not p.isRoot() and p.pageIsInPathTo(cur_page) and not last:
                tmp += p.createMenu(cur_page, found)

        return tmp + "
\n"

    def pageIsInPathTo(self, dest):
        parent = dest
        while parent:
            if parent == self:
                return True
            parent = parent.getParent()

        return False

class OtherFile:
    """ Copy other file """
    def __init__(self, src_path_root, src_path_rel, dest_dir=None):
        """ src_path_root / src_path_rel ==> dest_dir / src_path_rel """
        self._src_path_root = src_path_root
        self._src_path_rel = src_path_rel
        self._dest_dir = dest_dir

    def copy(self, site, to=None):
        if not to:
            to = self._dest_dir

        out_file = os.path.join(to, self._src_path_rel)
        out_dir = os.path.dirname(out_file)
        try:
            os.makedirs(out_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                fail(str(e))

        site.delFromFileIndex(out_file)
        shutil.copy(os.path.join(self._src_path_root, self._src_path_rel),
                    os.path.join(out_dir, os.path.basename(self._src_path_rel)))
<commit_msg>Fixed bug: catch OSError from makedir if directory xeists<commit_after>#!/usr/bin/python3

import errno
import os
import re
import shutil
from subprocess import Popen, PIPE
from stawebg.config import Config
from stawebg.helper import (listFolders, findFiles, findDirs, fail, matchList,
                            cleverCapitalize)

version = "0.1"

isFile = lambda f: os.path.isfile(f)
matchPath = lambda f, c: os.path.abspath(f)[len(os.path.abspath(c.getAbsSrcPath()))+1:]
isIndex = lambda f, site: matchList(matchPath(f, site), site.getConfig(['files', 'index']))
isCont = lambda f, site: matchList(matchPath(f, site), site.getConfig(['files', 'content']))
isExcluded = lambda f, site, c: matchList(matchPath(f, site), c.get(['files', 'exclude'], False, []))
isHidden = lambda f, site, c: matchList(matchPath(f, site), c.get(['files', 'hidden'], False, []))

class Project:
    def __init__(self, project_dir="", test=False):
        self._sites = []
        self._layouts = {}
        self._root_dir = project_dir
        self._test = test

        self._config = Config(os.path.join(self._root_dir, "stawebg.json"),
                              Config.global_struct)

        # Make directories absolute
        dirs = self._config.get(["dirs"])
        for k in dirs:
            dirs[k] = os.path.join(self._root_dir, dirs[k])

    def read(self):
        self._readLayouts()
        self._readSites()

    def copy(self):
        for s in self._sites:
            s.copy()

    def getConfig(self, key, fail=True, default=None):
        return self._config.get(key, fail, default)

    def getLayout(self, name=None):
        if not name:
            name = "default"

        layout = self._layouts.get(name)

        if not layout:
            fail("Can't find layout: " + name)

        return layout

    def getOutputDir(self):
        if self._test:
            return self.getConfig(["dirs", "test"])
        else:
            return self.getConfig(["dirs", "out"])

    # Add all layouts to list
    def _readLayouts(self):
        for name in listFolders(self.getConfig(['dirs', 'layouts'])):
            self._layouts[name] = Layout(self, name)
            self._layouts[name].read()

    # Add all site directories to list
    def _readSites(self):
        for s in listFolders(self.getConfig(['dirs', 'sites'])):
            site = Site(s, self)
            self._sites.append(site)
            site.read()

class Layout:
    def __init__(self, project, name):
        self._project = project
        self._name = name
        self._dir = os.path.join(self._project.getConfig(['dirs', 'layouts']),
                                 name)
        self._template = os.path.join(self._dir, 'template.html')
        self._other_files = []

        print("Found layout: " + self._name)

    def read(self):
        for f in findFiles(self._dir, [".html"]):
            print("\tFound file: " + f)
            self._other_files.append(OtherFile(self._dir,
                                               os.path.relpath(f, self._dir)))

    def copy(self, dest, site):
        for f in self._other_files:
            f.copy(site, os.path.join(dest, self.getSubdir()))

    def getSubdir(self):
        return os.path.join("style", self._name)

    def getTemplate(self):
        return self._template

class Site:
    def __init__(self, name, project):
        self._project = project
        self._name = name
        self._root = None
        self._other_files = []
        self._config = self._project._config
        self._layouts = []
        self._file_index = []
        print("Found site: " + self._name)

    def getConfig(self, key, fail=True, default=None):
        return self._config.get(key, fail, default)

    def getAbsSrcPath(self):
        return os.path.join(self.getConfig(['dirs', "sites"]), self._name)

    def getAbsDestPath(self):
        return os.path.join(self._project.getOutputDir(), self._name)

    def getProject(self):
        return self._project

    def addLayout(self, name):
        layout = self._project.getLayout(name)
        if layout not in self._layouts:
            self._layouts.append(layout)

    def getSiteTitle(self):
        tmp = self.getConfig(["title"], False)
        if tmp:
            return tmp

        return self._name

    def getSiteSubtitle(self):
        tmp = self.getConfig(["subtitle"], False)
        if tmp:
            return tmp

        return ""

    def read(self):
        self._readConfig()
        self._createFileIndex()
        self._readHelper(self.getAbsSrcPath(), self._root)

    def copy(self):
        print("Create site: " + self._name)

        # Pages
        self._root.copy()

        # Layouts
        for l in self._layouts:
            l.copy(self.getAbsDestPath(), self)

        # Other files
        for f in self._other_files:
            f.copy(self)

        # Cleanup
        self._cleanupOutput()

    def _readConfig(self):
        filename = os.path.join(self.getConfig(["dirs", "sites"]),
                                self._name + ".json")

        if not os.path.isfile(filename):
            fail("Can't find config file: " + filename)

        site_config = Config(filename, Config.site_struct)
        self._config = Config.merge(self._config, site_config, True)

    def _readHelper(self, dir_path, parent, dir_hidden=False, page_config=None):
        index_rename = None
        if page_config:
            index_rename = page_config.get(["files", "rename",
                                            os.path.basename(dir_path)], False)
            page_config.delete(["files", "sort"], False)
            page_config.delete(["files", "rename"], False)
        else:
            page_config = self._config

        entries = sorted(os.listdir(dir_path))

        if "stawebg.json" in entries:
            tmp_config = Config(os.path.join(dir_path, "stawebg.json"),
                                Config.directory_struct)
            page_config = Config.merge(page_config, tmp_config, False)

        # Add layout to list -> copy later
        self.addLayout(page_config.get(["layout"], False))

        # First we have to find the index file in this directory…
        idx = None
        for f in entries:
            absf = os.path.join(dir_path, f)
            if isFile(absf) and isCont(absf, self) and isIndex(absf, self):
                if index_rename:
                    page_config.add(["files", "rename", f], index_rename)
                idx = Page(os.path.split(dir_path)[1], absf, self, parent,
                           dir_hidden or isHidden(absf, self, page_config),
                           page_config)
                entries.remove(f)
                break
        # …or create an empty page as index
        if not idx:
            dirname = os.path.split(dir_path)[1]
            if index_rename:
                page_config.add(["files", "rename", None], index_rename)
            idx = Page(dirname, None, self, parent, dir_hidden or
                       isHidden(dirname, self, page_config), page_config)

        if parent:
            parent.appendPage(idx)
        else:
            self._root = idx

        # Sort entries as specified in configuration
        sorted_entries = page_config.get(["files", "sort"], False, [])
        for s in reversed(sorted_entries):
            absf = os.path.join(dir_path, s)
            if not s in entries:
                print("\tFile not found (specified in sort): " + absf)
            else:
                entries.remove(s)
                entries.insert(0, s)

        # Make absolute paths and check if it's a page
        for f in entries:
            absf = os.path.join(dir_path, f)
            hidden = dir_hidden or isHidden(absf, self, page_config)

            # Content file -> Page
            if isFile(absf) and isCont(absf, self):
                print("\tFound page: " + absf)
                idx.appendPage(Page(os.path.splitext(f)[0], absf, self, idx,
                                    hidden, page_config))
            # Directory -> Go inside
            elif os.path.isdir(absf):
                self._readHelper(absf, idx, hidden, page_config.copy())
            # Unknown object
            else:
                if not isExcluded(absf, self, page_config):
                    tmp = OtherFile(self.getAbsSrcPath(),
                                    os.path.relpath(
                                        absf, self.getAbsSrcPath()),
                                    self.getAbsDestPath())
                    self._other_files.append(tmp)
                    print("\tFound unkown object: " + absf)

    def createMenu(self, cur_page):
        return self._root.createMenu(cur_page)

    def _createFileIndex(self):
        path = self.getAbsDestPath()
        if os.path.isdir(path):
            self._file_index = findFiles(path)

    def delFromFileIndex(self, path):
        if path in self._file_index:
            self._file_index.remove(path)

    def _rmFileIndex(self):
        for f in self._file_index:
            print("\tRemove old file: " + f)
            try:
                os.remove(f)
            except OSError as e:
                print("\tError: " + str(e))

    def _cleanupOutput(self):
        self._rmFileIndex()

        # Delete empty directories
        for d in findDirs(self.getAbsDestPath()):
            if not os.listdir(d):
                print("\tRemove empty directory: " + d)
                try:
                    os.rmdir(d)
                except OSError as e:
                    print("\tError: " + str(e))

class Page:
    def __init__(self, name, absPath, site, parent, hidden, config):
        self._name = name
        self._absSrc = absPath
        self._site = site
        self._hidden = hidden
        self._parent = parent
        self._subpages = []
        self._config = config

        self._site.delFromFileIndex(self._getDestFile())

    def appendPage(self, p):
        self._subpages.append(p)

    def getPages(self):
        return self._subpages

    def getParent(self):
        return self._parent

    def getName(self):
        return self._name

    def isRoot(self):
        return self._parent is None

    def isHidden(self):
        return self._hidden

    def getLayout(self):
        layout = self._config.get(["layout"], False)
        return self._site.getProject().getLayout(layout)

    def copy(self):
        # Use 'codecs' package to support UTF-8?
        try:
            os.makedirs(os.path.dirname(self._getDestFile()))
            outf = open(self._getDestFile(), 'w')
            tplf = open(self.getLayout().getTemplate())
            outf.write(self._replaceKeywords(tplf.read()))
            tplf.close()
            outf.close()
        # TODO: check for other possible exceptions
        except OSError as e:
            if e.errno != errno.EEXIST:
                fail(str(e))
        except IOError as e:
            fail("Error creating " + self._getDestFile() + ": " + str(e))

        # Copy subpages
        for p in self._subpages:
            p.copy()

    def _translateMarkup(self):
        text = ''

        if not self._absSrc:
            return text

        with open(self._absSrc, "rt") as src:
            text = src.read()

        config = self._site.getConfig(["markup"], False)
        if not config:
            return text
        tool = config.get(os.path.splitext(self._absSrc)[1])

        if tool:
            pipe = Popen(tool, stdin=PIPE, stdout=PIPE, stderr=PIPE)
            out, err = pipe.communicate(text.encode())
            if len(err):
                fail(' '.join(tool) + ": " + err.decode())
            return out.decode()
        return text

    def _replaceKeywords(self, text):
        reps = {"%ROOT%": self.getRootLink(),
                "%CUR%": self.getCurrentLink(),
                "%LAYOUT%": self.getLayoutDir(),
                "%TITLE%": self.getTitle(),
                "%SITETITLE%": self._site.getSiteTitle(),
                "%SITESUBTITLE%": self._site.getSiteSubtitle(),
                "%MENU%": self._site.createMenu(self)}
        trans = lambda m: reps[m.group(0)]
        rc = re.compile('|'.join(map(re.escape, reps)))
        return rc.sub(trans, text.replace("%CONTENT%", self._translateMarkup()))

    def _getDestFile(self):
        tmp_path = ""

        parent = self.getParent()
        while parent and not parent.isRoot():
            tmp_path = os.path.join(parent.getName(), tmp_path)
            parent = parent.getParent()

        if not self.isRoot():
            tmp_path = os.path.join(tmp_path, self.getName())

        return os.path.join(self._site.getAbsDestPath(), tmp_path,
                            "index.html")

    def getRootLink(self):
        tmp = ""

        parent = self.getParent()
        while parent:
            tmp = tmp + "../"
            parent = parent.getParent()

        return tmp

    def getCurrentLink(self):
        return '' if self._absSrc and isIndex(self._absSrc, self._site) else '../'

    def getLayoutDir(self):
        tmp = self.getRootLink()
        if len(tmp):
            tmp += "/"
        tmp += self.getLayout().getSubdir() + "/"
        return tmp

    def getLink(self, origin=None):
        tmp = ""
        parent = self
        while parent and not parent.isRoot():
            tmp = parent.getName() + "/" + tmp
            parent = parent.getParent()

        if not origin:
            origin = self

        tmp = origin.getRootLink() + tmp

        if tmp == "":
            tmp = "."

        return tmp

    def getShortTitle(self):
        key = None
        if self._absSrc:
            key = os.path.basename(self._absSrc)

        rename = self._config.get(["files", "rename", key], False)

        if rename:
            return rename
        elif not self.getParent():
            return "Home"
        else:
            return cleverCapitalize(self.getName())

    def getTitle(self, no_home=False):
        if not self.getParent():
            if no_home:
                return self._site.getSiteTitle()
            else:
                return self._site.getSiteTitle() + " > " + self.getShortTitle()
        else:
            return self.getParent().getTitle(True) + " > " + self.getShortTitle()

    def createMenu(self, cur_page, last=False):
        items = self._subpages[:]

        # Add root link
        if self.isRoot():
            items.insert(0, self)

        # Create HTML Code
        found = False
        tmp = "
\n"
        for p in items:
            if p == cur_page:
                found = True

            if p.isHidden():
                continue

            active = ""
            if p.pageIsInPathTo(cur_page) and (not p.isRoot() or p == cur_page):
                active = " class=\"active\""

            tmp = ''.join([tmp, "
", p.getShortTitle(), "
\n"])

            # Create submenu
            if not p.isRoot() and p.pageIsInPathTo(cur_page) and not last:
                tmp += p.createMenu(cur_page, found)

        return tmp + "
\n"

    def pageIsInPathTo(self, dest):
        parent = dest
        while parent:
            if parent == self:
                return True
            parent = parent.getParent()

        return False

class OtherFile:
    """ Copy other file """
    def __init__(self, src_path_root, src_path_rel, dest_dir=None):
        """ src_path_root / src_path_rel ==> dest_dir / src_path_rel """
        self._src_path_root = src_path_root
        self._src_path_rel = src_path_rel
        self._dest_dir = dest_dir

    def copy(self, site, to=None):
        if not to:
            to = self._dest_dir

        out_file = os.path.join(to, self._src_path_rel)
        out_dir = os.path.dirname(out_file)
        try:
            os.makedirs(out_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                fail(str(e))

        site.delFromFileIndex(out_file)
        shutil.copy(os.path.join(self._src_path_root, self._src_path_rel),
                    os.path.join(out_dir, os.path.basename(self._src_path_rel)))
<|endoftext|>


<commit_before>__version_info__ = (0, 1, 3)
__version__ = '.'.join(str(_) for _ in __version_info__)
<commit_msg>Bump version.<commit_after>__version_info__ = (0, 1, 4)
__version__ = '.'.join(str(_) for _ in __version_info__)
<|endoftext|>


<commit_before>import argparse
import os
import sys
from io import StringIO

from tabulate import tabulate

from pch2csd import __version__, __homepage__
from pch2csd.csdgen import ZakSpace, Csd, UdoTemplate, UdoTemplateValidation
from pch2csd.parse import parse_pch2
from pch2csd.patch import Patch
from pch2csd.resources import get_template_module_path, ProjectData

def get_test_resource(path: str) -> str:
    return os.path.abspath(
        os.path.join(os.path.dirname(__file__), '..', 'tests', 'resources', path))

def _all_modules_implemented(patch: Patch):
    not_implemented = [x.type_name for x in patch.modules
                       if not os.path.isfile(get_template_module_path(x.type))]
    if len(not_implemented) > 0:
        print('The patch file contains some modules that has not been implemented yet:')
        print(', '.join(not_implemented))
        print('Please, consider contributing these modules, following our tutorial:')
        print('https://github.com/gleb812/pch2csd/wiki/How-to-add-new-modules')
        return False
    return True

def validate_udo(type_id: int, io=sys.stdout, print_action=True):
    if print_action:
        print("checking module type '{id}' ({id}.txt)".format(id=type_id),
              file=io)
    pch2_files = [get_test_resource(s) for s in ['test_all_modules_1.pch2',
                                                 'test_all_modules_2.pch2']]
    data, mod, patch = ProjectData(), None, None
    for p in map(lambda x: parse_pch2(data, x), pch2_files):
        for m in p.modules:
            if m.type == type_id:
                mod, patch = m, p
                break
    if mod is not None:
        if print_action:
            print('module name: {}'.format(mod.type_name), file=io)
        udo = UdoTemplate(mod)
        v = UdoTemplateValidation(data, udo)
        v.print_errors(io)
        return v.is_valid(with_todos=True)
    else:
        print("error: unknown module type '{}'".format(type_id), file=io)
        return False

def print_pch2(fn: str):
    if not fn.lower().endswith('.pch2'):
        print("error: patch file should have extension '.pch2'")
        exit(-1)
    data = ProjectData()
    path = os.path.abspath(fn)
    patch = parse_pch2(data, path)

    mod_table = [['Name', 'ID', 'Type', 'Parameters', 'Area']]
    for m in patch.modules:
        p = patch.find_mod_params(m.location, m.id)
        mod_table.append([m.type_name,
                          m.id,
                          m.type,
                          str(p.values),
                          m.location.short_str()])
    cab_table = [['From', '', 'To', 'Color', 'Type', 'Area']]
    for c in patch.cables:
        mf_name = patch.find_module(c.module_from, c.loc).type_name
        mt_name = patch.find_module(c.module_to, c.loc).type_name
        pin1, pin2 = c.type.short_str().split('-')
        cab_table.append([
            '{}(id={}, {}={})'.format(mf_name, c.module_from, pin1, c.jack_from),
            '->',
            '{}(id={}, {}={})'.format(mt_name, c.module_to, pin2, c.jack_to),
            c.color.short_str(),
            c.type.short_str(),
            c.loc.short_str()])
    print('Patch file: {}\n'.format(os.path.basename(path)))
    print('Modules')
    print(tabulate(mod_table, headers='firstrow', tablefmt='simple'))
    print('\nCables')
    print(tabulate(cab_table, headers='firstrow', tablefmt='simple'))

def convert_pch2(fn: str):
    if not fn.lower().endswith('.pch2'):
        print("error: the patch file should have extension '.pch2'")
        exit(-1)
    data = ProjectData()
    path = os.path.abspath(os.path.expanduser(fn))
    p = parse_pch2(data, path)
    zak = ZakSpace()
    try:
        udos = zak.connect_patch(p)
    except ValueError:
        exit(-1)
    csd = Csd(p, zak, udos)
    dirname = os.path.dirname(path)
    csd_save_path = os.path.join(dirname, os.path.basename(path) + '.csd')
    with open(csd_save_path, 'w') as f:
        f.write(csd.get_code())

def gen_udo_status_doc():
    tpl_url = 'https://github.com/gleb812/pch2csd/blob/master/pch2csd/resources/templates/modules/{}.txt'
    data = ProjectData()
    with open('Module-implementation-status.md', 'w') as md:
        md.write('This file is automatically generated.\n\n')
        md.write('| Template | Module name | Status |\n')
        md.write('|----------|-------------|--------|\n')
        for p in [parse_pch2(data, get_test_resource(pch2file)) for pch2file
                  in ['test_all_modules_1.pch2', 'test_all_modules_2.pch2']]:
            for m in p.modules:
                status = StringIO()
                validate_udo(m.type, status, print_action=False)
                md.write('| [{}]({}) | {} | {} |\n'.format(
                    '{}.txt'.format(m.type),
                    tpl_url.format(m.type),
                    m.type_name,
                    '<br>'.join(status.getvalue().splitlines())))

def main():
    arg_parser = argparse.ArgumentParser(
        prog='pch2csd',
        description='convert Clavia Nord Modular G2 patches to the Csound code',
        epilog='Version {}, homepage: {}'.format(__version__, __homepage__))
    arg_parser.add_argument('arg', metavar='arg', nargs='?', default='patch.pch2',
                            help='a pch2 file path or an UDO numerical ID')
    group = arg_parser.add_mutually_exclusive_group()
    group.add_argument('-p', '--print', action='store_const', const=True,
                       help='parse the patch file and print its content')
    group.add_argument('-c', '--check-udo', action='store_const', const=True,
                       help="validate the UDO template file (overrides '-p')")
    group.add_argument('-v', '--version', action='version',
                       version='%(prog)s ' + __version__)
    group.add_argument('-e', action='store_const', const=True,
                       help='show the elephant and exit')
    args = arg_parser.parse_args()
    if args.check_udo:
        try:
            type_id = int(args.arg[0])
            validate_udo(type_id)
        except ValueError:
            print("you should pass the integer as the 'arg' parameter when using '--check-udo'")
    elif args.print:
        print_pch2(args.arg[0])
    elif args.e:
        show_elephant()
    else:
        if args.arg[0] == 'gen_udo_status_doc':
            gen_udo_status_doc()
        else:
            convert_pch2(args.arg[0])

def show_elephant():
    print('///////////////////osyyo///////////////////////////////////////////////////////')
    print('//////////////+oshmNMMMmNmhyso+//////////////////+++++////////////////////+o///')
    print('///////////+oshydNMMMMMMMMMMMNh++++++++++ossssyysssyshhys+//////////////+hNmhys')
    print('/////////+oydmmNNNNNNNNNNNMMNNdhyyyyyyyhhddy+++::/ooossyyhs+///////////omMMMNNN')
    print('///////+oyyhhhdhhhhhhdmdmmddhyshhyysys++ossys+--+syyyyyysoo++/////////+hmmmmmdy')
    print('///+++++++++ooooooosossssoo+++syyyssyyss+-..`.ydmmddyo+/+++/++++++++++shhhhhyys')
    print('+++                        oooyhyyhyyyhhdso+/:sddyo+//++/////++++++++++ooosssss')
    print('+++ Clavia Nord Modular G2 sshhhyyyyyys+-+hho/ys+///++/////:+++++++++++++++++++')
    print('+++     Patch Converter    ooossosyyy+:``.--`.//+/+/://+/o+++++++++++++++++++++')
    print('+++                        oo+oysysso/:-.``````.-:/+/-/+syso+++++++++++++++++++')
    print('++oooooooooooooooooooooooooooosssysoosys+-``` ``-:////://oosooooooooo++++++++++')
    print('ooooooooooooosssssooosssssssssshyyso+shdh.`    `-/:-:-:--/++ooooooooooooooooyso')
    print('ssssssssyyyyyyyyyyyyyyyyyssssooooso+++yhh-     .:/--````-::-/oooooooooooosyhhdd')
    print('ossosssssssssssssssssssssssss/++++/--/+hs`   `.`-...````-..`oooooooosssyssssyyy')
    print('ooooooosssssssssssssssyysssss/////-`  sNm     `   .`   ``` /oooosoosyhdhysooyhd')
    print('oooooosssssssssssssshdyysssym/:::-`/.:mmo         `       :sssssyyyyyysoosyyyyy')
    print('osssssssssssssssyyhdy+++shmNs-.```.Ny``                   +ssssyyyhhyyyyyssssoo')
    print('sssshddhysyssyyyhdds/oyhsdysh-.   omm-       -.          :.+yssssyyyyysyhyyysyh')
    print('yhhhdhhhhhyyyyyhhhh/.:ysydmNh.   .hmmy      `:-`  ``    `yo-ohyyyyyyyyyyyysssss')
    print('syyyyyyyyyyhddhddmmy.`.:/++o/`  `yhhdh.     ..` ````    ohhyyyyyyyyyyyyyyyyyyss')
    print('hysyyyyhhhyhhhhhyyhhy/`  `-:. `/yyhhhyo    `.` ``      +yyyyyyyyyyyyysyssssssss')
    print('hyyhhyyhhdhhhhyyyyyyyyyo+///+syyyyyyyhy-   ..``:yo`   :hhyyyyysyyyssyysssssssss')
    print('yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyo  .-.``syy-   syyyyyyssssssssssssssssss')
    print('ssssyyyyyyyyyyyyyyyysssssosssoooooooooos-`--.`-sss.  `ssssooooooooooooooooooooo')
    print('sssyyysssssssssssssooooooooossssssssssss+-:-.`oysy   -yyyyyyyyyyyyyyyyyysyysyyy')
    print('yyyyyyyyyyyyyhhhhyhhhhhdhhdddddddhdddddd/.:-``yddy   :ddddhhdddddddddhhhhhhdhhh')
    print('hhddddddddddddddddddddddmmmmmmmdddmmdddh/.:.../hd+   .ddddddddddddhhhyhhhhhhhhh')
    print('hhhhhhhhhhhhdddhdddddddddhhhhhhhhdhhhhhh-.o+/--hy.    -osyhhhddddhhhhyyyyyyyyys')
    print('dddhhhhhhhhhhhhdddhdhhhhhhhhhhyyyssyo//:`-hyys:` ```.-::/+osyyysyyyyyyyyyhyyhys')
    print('hhhyyhhhhhdhddhhhhyyysosoysosooo//:----.`+yysssssossooyyysyhhhhyhhyyyyyyhhyyyyy')

if __name__ == '__main__':
    main()
<commit_msg>[fix] incorrect input patch name handling<commit_after>import argparse
import os
import sys
from io import StringIO

from tabulate import tabulate

from pch2csd import __version__, __homepage__
from pch2csd.csdgen import ZakSpace, Csd, UdoTemplate, UdoTemplateValidation
from pch2csd.parse import parse_pch2
from pch2csd.patch import Patch
from pch2csd.resources import get_template_module_path, ProjectData

def get_test_resource(path: str) -> str:
    return os.path.abspath(
        os.path.join(os.path.dirname(__file__), '..', 'tests', 'resources', path))

def _all_modules_implemented(patch: Patch):
    not_implemented = [x.type_name for x in patch.modules
                       if not os.path.isfile(get_template_module_path(x.type))]
    if len(not_implemented) > 0:
        print('The patch file contains some modules that has not been implemented yet:')
        print(', '.join(not_implemented))
        print('Please, consider contributing these modules, following our tutorial:')
        print('https://github.com/gleb812/pch2csd/wiki/How-to-add-new-modules')
        return False
    return True

def validate_udo(type_id: int, io=sys.stdout, print_action=True):
    if print_action:
        print("checking module type '{id}' ({id}.txt)".format(id=type_id),
              file=io)
    pch2_files = [get_test_resource(s) for s in ['test_all_modules_1.pch2',
                                                 'test_all_modules_2.pch2']]
    data, mod, patch = ProjectData(), None, None
    for p in map(lambda x: parse_pch2(data, x), pch2_files):
        for m in p.modules:
            if m.type == type_id:
                mod, patch = m, p
                break
    if mod is not None:
        if print_action:
            print('module name: {}'.format(mod.type_name), file=io)
        udo = UdoTemplate(mod)
        v = UdoTemplateValidation(data, udo)
        v.print_errors(io)
        return v.is_valid(with_todos=True)
    else:
        print("error: unknown module type '{}'".format(type_id), file=io)
        return False

def print_pch2(fn: str):
    if not fn.lower().endswith('.pch2'):
        print("error: patch file should have extension '.pch2'")
        exit(-1)
    data = ProjectData()
    path = os.path.abspath(fn)
    patch = parse_pch2(data, path)

    mod_table = [['Name', 'ID', 'Type', 'Parameters', 'Area']]
    for m in patch.modules:
        p = patch.find_mod_params(m.location, m.id)
        mod_table.append([m.type_name,
                          m.id,
                          m.type,
                          str(p.values),
                          m.location.short_str()])
    cab_table = [['From', '', 'To', 'Color', 'Type', 'Area']]
    for c in patch.cables:
        mf_name = patch.find_module(c.module_from, c.loc).type_name
        mt_name = patch.find_module(c.module_to, c.loc).type_name
        pin1, pin2 = c.type.short_str().split('-')
        cab_table.append([
            '{}(id={}, {}={})'.format(mf_name, c.module_from, pin1, c.jack_from),
            '->',
            '{}(id={}, {}={})'.format(mt_name, c.module_to, pin2, c.jack_to),
            c.color.short_str(),
            c.type.short_str(),
            c.loc.short_str()])
    print('Patch file: {}\n'.format(os.path.basename(path)))
    print('Modules')
    print(tabulate(mod_table, headers='firstrow', tablefmt='simple'))
    print('\nCables')
    print(tabulate(cab_table, headers='firstrow', tablefmt='simple'))

def convert_pch2(fn: str):
    if not fn.lower().endswith('.pch2'):
        print("error: the patch file should have extension '.pch2'")
        exit(-1)
    data = ProjectData()
    path = os.path.abspath(os.path.expanduser(fn))
    p = parse_pch2(data, path)
    zak = ZakSpace()
    try:
        udos = zak.connect_patch(p)
    except ValueError:
        exit(-1)
    csd = Csd(p, zak, udos)
    dirname = os.path.dirname(path)
    csd_save_path = os.path.join(dirname, os.path.basename(path) + '.csd')
    with open(csd_save_path, 'w') as f:
        f.write(csd.get_code())

def gen_udo_status_doc():
    tpl_url = 'https://github.com/gleb812/pch2csd/blob/master/pch2csd/resources/templates/modules/{}.txt'
    data = ProjectData()
    with open('Module-implementation-status.md', 'w') as md:
        md.write('This file is automatically generated.\n\n')
        md.write('| Template | Module name | Status |\n')
        md.write('|----------|-------------|--------|\n')
        for p in [parse_pch2(data, get_test_resource(pch2file)) for pch2file
                  in ['test_all_modules_1.pch2', 'test_all_modules_2.pch2']]:
            for m in p.modules:
                status = StringIO()
                validate_udo(m.type, status, print_action=False)
                md.write('| [{}]({}) | {} | {} |\n'.format(
                    '{}.txt'.format(m.type),
                    tpl_url.format(m.type),
                    m.type_name,
                    '<br>'.join(status.getvalue().splitlines())))

def main():
    arg_parser = argparse.ArgumentParser(
        prog='pch2csd',
        description='convert Clavia Nord Modular G2 patches to the Csound code',
        epilog='Version {}, homepage: {}'.format(__version__, __homepage__))
    arg_parser.add_argument('arg', metavar='arg', nargs='?', default='patch.pch2',
                            help='a pch2 file path or an UDO numerical ID')
    group = arg_parser.add_mutually_exclusive_group()
    group.add_argument('-p', '--print', action='store_const', const=True,
                       help='parse the patch file and print its content')
    group.add_argument('-c', '--check-udo', action='store_const', const=True,
                       help="validate the UDO template file (overrides '-p')")
    group.add_argument('-v', '--version', action='version',
                       version='%(prog)s ' + __version__)
    group.add_argument('-e', action='store_const', const=True,
                       help='show the elephant and exit')
    args = arg_parser.parse_args()
    if args.check_udo:
        try:
            type_id = int(args.arg)
            validate_udo(type_id)
        except ValueError:
            print("you should pass the integer as the 'arg' parameter when using '--check-udo'")
    elif args.print:
        print_pch2(args.arg)
    elif args.e:
        show_elephant()
    else:
        if args.arg[0] == 'gen_udo_status_doc':
            gen_udo_status_doc()
        else:
            convert_pch2(args.arg)

def show_elephant():
    print('///////////////////osyyo///////////////////////////////////////////////////////')
    print('//////////////+oshmNMMMmNmhyso+//////////////////+++++////////////////////+o///')
    print('///////////+oshydNMMMMMMMMMMMNh++++++++++ossssyysssyshhys+//////////////+hNmhys')
    print('/////////+oydmmNNNNNNNNNNNMMNNdhyyyyyyyhhddy+++::/ooossyyhs+///////////omMMMNNN')
    print('///////+oyyhhhdhhhhhhdmdmmddhyshhyysys++ossys+--+syyyyyysoo++/////////+hmmmmmdy')
    print('///+++++++++ooooooosossssoo+++syyyssyyss+-..`.ydmmddyo+/+++/++++++++++shhhhhyys')
    print('+++                        oooyhyyhyyyhhdso+/:sddyo+//++/////++++++++++ooosssss')
    print('+++ Clavia Nord Modular G2 sshhhyyyyyys+-+hho/ys+///++/////:+++++++++++++++++++')
    print('+++     Patch Converter    ooossosyyy+:``.--`.//+/+/://+/o+++++++++++++++++++++')
    print('+++                        oo+oysysso/:-.``````.-:/+/-/+syso+++++++++++++++++++')
    print('++oooooooooooooooooooooooooooosssysoosys+-``` ``-:////://oosooooooooo++++++++++')
    print('ooooooooooooosssssooosssssssssshyyso+shdh.`    `-/:-:-:--/++ooooooooooooooooyso')
    print('ssssssssyyyyyyyyyyyyyyyyyssssooooso+++yhh-     .:/--````-::-/oooooooooooosyhhdd')
    print('ossosssssssssssssssssssssssss/++++/--/+hs`   `.`-...````-..`oooooooosssyssssyyy')
    print('ooooooosssssssssssssssyysssss/////-`  sNm     `   .`   ``` /oooosoosyhdhysooyhd')
    print('oooooosssssssssssssshdyysssym/:::-`/.:mmo         `       :sssssyyyyyysoosyyyyy')
    print('osssssssssssssssyyhdy+++shmNs-.```.Ny``                   +ssssyyyhhyyyyyssssoo')
    print('sssshddhysyssyyyhdds/oyhsdysh-.   omm-       -.          :.+yssssyyyyysyhyyysyh')
    print('yhhhdhhhhhyyyyyhhhh/.:ysydmNh.   .hmmy      `:-`  ``    `yo-ohyyyyyyyyyyyysssss')
    print('syyyyyyyyyyhddhddmmy.`.:/++o/`  `yhhdh.     ..` ````    ohhyyyyyyyyyyyyyyyyyyss')
    print('hysyyyyhhhyhhhhhyyhhy/`  `-:. `/yyhhhyo    `.` ``      +yyyyyyyyyyyyysyssssssss')
    print('hyyhhyyhhdhhhhyyyyyyyyyo+///+syyyyyyyhy-   ..``:yo`   :hhyyyyysyyyssyysssssssss')
    print('yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyo  .-.``syy-   syyyyyyssssssssssssssssss')
    print('ssssyyyyyyyyyyyyyyyysssssosssoooooooooos-`--.`-sss.  `ssssooooooooooooooooooooo')
    print('sssyyysssssssssssssooooooooossssssssssss+-:-.`oysy   -yyyyyyyyyyyyyyyyyysyysyyy')
    print('yyyyyyyyyyyyyhhhhyhhhhhdhhdddddddhdddddd/.:-``yddy   :ddddhhdddddddddhhhhhhdhhh')
    print('hhddddddddddddddddddddddmmmmmmmdddmmdddh/.:.../hd+   .ddddddddddddhhhyhhhhhhhhh')
    print('hhhhhhhhhhhhdddhdddddddddhhhhhhhhdhhhhhh-.o+/--hy.    -osyhhhddddhhhhyyyyyyyyys')
    print('dddhhhhhhhhhhhhdddhdhhhhhhhhhhyyyssyo//:`-hyys:` ```.-::/+osyyysyyyyyyyyyhyyhys')
    print('hhhyyhhhhhdhddhhhhyyysosoysosooo//:----.`+yysssssossooyyysyhhhhyhhyyyyyyhhyyyyy')

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import requests,json,re,csv,os,subprocess,urllib2,argparse,getpass
from pprint import pprint
from os.path import expanduser
from urllib2 import Request, urlopen
from clip_idlist import idlist
from clip_geojson import geojsonc
from cli_aoi2json import aoijson
from clip_json import jsonc
from clip_download import downloadclips
from cli_sorter import sort
os.chdir(os.path.dirname(os.path.realpath(__file__)))
def planet_key_entry():
    try:
        subprocess.call('planet init',shell=True)
    except Exception as e:
        print(e)
def planet_key_from_parser(args):
    planet_key_entry()
def planet_quota():
    try:
        subprocess.call('python planet_quota.py',shell=True)
    except Exception as e:
        print(e)
def planet_quota_from_parser(args):
    planet_quota()
def aoijson_from_parser(args):
    aoijson(start=args.start,end=args.end,cloud=args.cloud,inputfile=args.inputfile,geo=args.geo,loc=args.loc)
def activate_from_parser(args):
    aoi_json=str(args.aoi)
    action_planet=str(args.action)
    asset_type=str(args.asset)
    subprocess.call("python download.py --query "+'"'+aoi_json+'" '+"--activate "+asset_type,shell=True)
##    try:
##        os.system("python download.py --query "+args.aoi+" --"+args.action+" "+asset_type)
##    except Exception:
##        print(' ')
def idlist_from_parser(args):
    idlist(aoi=args.aoi,item_asset=args.asset)
def geojsonc_from_parser(args):
    geojsonc(path=args.path,item=args.item,asset=args.asset)
def jsonc_from_parser(args):
    jsonc(path=args.path,item=args.item,asset=args.asset)
def downloadclips_from_parser(args):
    downloadclips(filepath=args.dir)
def sort_from_parser(args):
    sort(zipped=args.zipped,unzipped=args.unzipped)

spacing="                               "
def main(args=None):
    parser = argparse.ArgumentParser(description='Planet Clip Tools CLI')

    subparsers = parser.add_subparsers()

    parser_pp3 = subparsers.add_parser(' ', help='-------------------------------------------')
    parser_P2 = subparsers.add_parser(' ', help='-----Choose from Planet Clip Tools-----')
    parser_pp4 = subparsers.add_parser(' ', help='-------------------------------------------')

    parser_planet_key = subparsers.add_parser('apikey', help='Enter your planet API Key')
    parser_planet_key.set_defaults(func=planet_key_from_parser)

    parser_planet_quota = subparsers.add_parser('quota', help='Prints your Planet Quota Details')
    parser_planet_quota.set_defaults(func=planet_quota_from_parser)

    parser_aoijson=subparsers.add_parser('aoijson',help='Tool to convert KML, Shapefile,WKT,GeoJSON or Landsat WRS PathRow file to AreaOfInterest.JSON file with structured query for use with Planet API 1.0')
    parser_aoijson.add_argument('--start', help='Start date in YYYY-MM-DD?')
    parser_aoijson.add_argument('--end', help='End date in YYYY-MM-DD?')
    parser_aoijson.add_argument('--cloud', help='Maximum Cloud Cover(0-1) representing 0-100')
    parser_aoijson.add_argument('--inputfile',help='Choose a kml/shapefile/geojson or WKT file for AOI(KML/SHP/GJSON/WKT) or WRS (6 digit RowPath Example: 023042)')
    parser_aoijson.add_argument('--geo', default='./map.geojson',help='map.geojson/aoi.kml/aoi.shp/aoi.wkt file')
    parser_aoijson.add_argument('--loc', help='Location where aoi.json file is to be stored')
    parser_aoijson.set_defaults(func=aoijson_from_parser)

    parser_activate=subparsers.add_parser('activate',help='Tool to query and/or activate Planet Assets')
    parser_activate.add_argument('--aoi', help='Choose aoi.json file created earlier')
    parser_activate.add_argument('--action', help='choose between check/activate')
    parser_activate.add_argument('--asset',help='Choose between planet asset types (PSOrthoTile analytic/PSOrthoTile analytic_dn/PSOrthoTile visual/PSScene4Band analytic/PSScene4Band analytic_dn/PSScene3Band analytic/PSScene3Band analytic_dn/PSScene3Band visual/REOrthoTile analytic/REOrthoTile visual')
    parser_activate.set_defaults(func=activate_from_parser)

    parser_idlist = subparsers.add_parser('idlist', help='Allows users to generate an id list for the selected item and asset type for example item_asset= PSOrthoTile analytic/PSScene3Band visual. This is used with the clip tool')
    parser_idlist.add_argument('--aoi', help='Input path to the structured json file from which we will generate the clips',default=None)
    parser_idlist.add_argument('--asset', help='Choose from asset type for example:"PSOrthoTile analytic"|"REOrthoTile analytic"',default=None)
    parser_idlist.set_defaults(func=idlist_from_parser)

    parser_geojsonc = subparsers.add_parser('geojsonc', help='Allows users to batch submit clipping request to the Planet Clip API using geometry in geojson file')
    parser_geojsonc.add_argument('--path', help='Path to the geojson file including filename',default=None)
    parser_geojsonc.add_argument('--item', help='Choose from item type for example:"PSOrthoTile","REOrthoTile"',default=None)
    parser_geojsonc.add_argument('--asset', help='Choose from asset type for example: "visual","analytic"',default=None)
    parser_geojsonc.set_defaults(func=geojsonc_from_parser)

    parser_jsonc = subparsers.add_parser('jsonc', help='Allows users to batch submit clipping request to the Planet Clip API using geometry in structured json file. This is preferred because the structured JSON allows the activate tool to stream line asset ids being requested and to extract geometry from the same file')
    parser_jsonc.add_argument('--path', help='Path to the json file including filename',default=None)
    parser_jsonc.add_argument('--item', help='Choose from item type for example:"PSOrthoTile","REOrthoTile"',default=None)
    parser_jsonc.add_argument('--asset', help='Choose from asset type for example: "visual","analytic"',default=None)
    parser_jsonc.set_defaults(func=jsonc_from_parser)

    parser_downloadclips = subparsers.add_parser('downloadclips', help='Allows users to batch download clipped assets post computation using a directory path(Requires you to first activate and run geojson or json tool)')
    parser_downloadclips.add_argument('--dir', help='Output directory to save the assets. All files are zipped and include metadata',default=None)
    parser_downloadclips.set_defaults(func=downloadclips_from_parser)

    parser_sort = subparsers.add_parser('sort', help='Allows users to unzip downloaded files to new folder and sorts into images and metadata')
    parser_sort.add_argument('--zipped', help='Folder containing downloaded clipped files which are zipped',default=None)
    parser_sort.add_argument('--unzipped', help='Folder where you want your files to be unzipped and sorted',default=None)
    parser_sort.set_defaults(func=sort_from_parser)

    args = parser.parse_args()

    args.func(args)

if __name__ == '__main__':
    main()
<commit_msg>Now handles urllib exceptions for py2 & py3<commit_after>import requests,json,re,csv,os,subprocess,urllib2,argparse,getpass
from pprint import pprint
from os.path import expanduser
from clip_idlist import idlist
from clip_geojson import geojsonc
from cli_aoi2json import aoijson
from clip_json import jsonc
from clip_download import downloadclips
from cli_sorter import sort
if sys.version_info > (3, 0):
    from urllib.request import urlopen, Request
else:
    from urllib2 import Request, urlopen
os.chdir(os.path.dirname(os.path.realpath(__file__)))
def planet_key_entry():
    try:
        subprocess.call('planet init',shell=True)
    except Exception as e:
        print(e)
def planet_key_from_parser(args):
    planet_key_entry()
def planet_quota():
    try:
        subprocess.call('python planet_quota.py',shell=True)
    except Exception as e:
        print(e)
def planet_quota_from_parser(args):
    planet_quota()
def aoijson_from_parser(args):
    aoijson(start=args.start,end=args.end,cloud=args.cloud,inputfile=args.inputfile,geo=args.geo,loc=args.loc)
def activate_from_parser(args):
    aoi_json=str(args.aoi)
    action_planet=str(args.action)
    asset_type=str(args.asset)
    subprocess.call("python download.py --query "+'"'+aoi_json+'" '+"--activate "+asset_type,shell=True)
##    try:
##        os.system("python download.py --query "+args.aoi+" --"+args.action+" "+asset_type)
##    except Exception:
##        print(' ')
def idlist_from_parser(args):
    idlist(aoi=args.aoi,item_asset=args.asset)
def geojsonc_from_parser(args):
    geojsonc(path=args.path,item=args.item,asset=args.asset)
def jsonc_from_parser(args):
    jsonc(path=args.path,item=args.item,asset=args.asset)
def downloadclips_from_parser(args):
    downloadclips(filepath=args.dir)
def sort_from_parser(args):
    sort(zipped=args.zipped,unzipped=args.unzipped)

spacing="                               "
def main(args=None):
    parser = argparse.ArgumentParser(description='Planet Clip Tools CLI')

    subparsers = parser.add_subparsers()

    parser_pp3 = subparsers.add_parser(' ', help='-------------------------------------------')
    parser_P2 = subparsers.add_parser(' ', help='-----Choose from Planet Clip Tools-----')
    parser_pp4 = subparsers.add_parser(' ', help='-------------------------------------------')

    parser_planet_key = subparsers.add_parser('apikey', help='Enter your planet API Key')
    parser_planet_key.set_defaults(func=planet_key_from_parser)

    parser_planet_quota = subparsers.add_parser('quota', help='Prints your Planet Quota Details')
    parser_planet_quota.set_defaults(func=planet_quota_from_parser)

    parser_aoijson=subparsers.add_parser('aoijson',help='Tool to convert KML, Shapefile,WKT,GeoJSON or Landsat WRS PathRow file to AreaOfInterest.JSON file with structured query for use with Planet API 1.0')
    parser_aoijson.add_argument('--start', help='Start date in YYYY-MM-DD?')
    parser_aoijson.add_argument('--end', help='End date in YYYY-MM-DD?')
    parser_aoijson.add_argument('--cloud', help='Maximum Cloud Cover(0-1) representing 0-100')
    parser_aoijson.add_argument('--inputfile',help='Choose a kml/shapefile/geojson or WKT file for AOI(KML/SHP/GJSON/WKT) or WRS (6 digit RowPath Example: 023042)')
    parser_aoijson.add_argument('--geo', default='./map.geojson',help='map.geojson/aoi.kml/aoi.shp/aoi.wkt file')
    parser_aoijson.add_argument('--loc', help='Location where aoi.json file is to be stored')
    parser_aoijson.set_defaults(func=aoijson_from_parser)

    parser_activate=subparsers.add_parser('activate',help='Tool to query and/or activate Planet Assets')
    parser_activate.add_argument('--aoi', help='Choose aoi.json file created earlier')
    parser_activate.add_argument('--action', help='choose between check/activate')
    parser_activate.add_argument('--asset',help='Choose between planet asset types (PSOrthoTile analytic/PSOrthoTile analytic_dn/PSOrthoTile visual/PSScene4Band analytic/PSScene4Band analytic_dn/PSScene3Band analytic/PSScene3Band analytic_dn/PSScene3Band visual/REOrthoTile analytic/REOrthoTile visual')
    parser_activate.set_defaults(func=activate_from_parser)

    parser_idlist = subparsers.add_parser('idlist', help='Allows users to generate an id list for the selected item and asset type for example item_asset= PSOrthoTile analytic/PSScene3Band visual. This is used with the clip tool')
    parser_idlist.add_argument('--aoi', help='Input path to the structured json file from which we will generate the clips',default=None)
    parser_idlist.add_argument('--asset', help='Choose from asset type for example:"PSOrthoTile analytic"|"REOrthoTile analytic"',default=None)
    parser_idlist.set_defaults(func=idlist_from_parser)

    parser_geojsonc = subparsers.add_parser('geojsonc', help='Allows users to batch submit clipping request to the Planet Clip API using geometry in geojson file')
    parser_geojsonc.add_argument('--path', help='Path to the geojson file including filename',default=None)
    parser_geojsonc.add_argument('--item', help='Choose from item type for example:"PSOrthoTile","REOrthoTile"',default=None)
    parser_geojsonc.add_argument('--asset', help='Choose from asset type for example: "visual","analytic"',default=None)
    parser_geojsonc.set_defaults(func=geojsonc_from_parser)

    parser_jsonc = subparsers.add_parser('jsonc', help='Allows users to batch submit clipping request to the Planet Clip API using geometry in structured json file. This is preferred because the structured JSON allows the activate tool to stream line asset ids being requested and to extract geometry from the same file')
    parser_jsonc.add_argument('--path', help='Path to the json file including filename',default=None)
    parser_jsonc.add_argument('--item', help='Choose from item type for example:"PSOrthoTile","REOrthoTile"',default=None)
    parser_jsonc.add_argument('--asset', help='Choose from asset type for example: "visual","analytic"',default=None)
    parser_jsonc.set_defaults(func=jsonc_from_parser)

    parser_downloadclips = subparsers.add_parser('downloadclips', help='Allows users to batch download clipped assets post computation using a directory path(Requires you to first activate and run geojson or json tool)')
    parser_downloadclips.add_argument('--dir', help='Output directory to save the assets. All files are zipped and include metadata',default=None)
    parser_downloadclips.set_defaults(func=downloadclips_from_parser)

    parser_sort = subparsers.add_parser('sort', help='Allows users to unzip downloaded files to new folder and sorts into images and metadata')
    parser_sort.add_argument('--zipped', help='Folder containing downloaded clipped files which are zipped',default=None)
    parser_sort.add_argument('--unzipped', help='Folder where you want your files to be unzipped and sorted',default=None)
    parser_sort.set_defaults(func=sort_from_parser)

    args = parser.parse_args()

    args.func(args)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>
from utils import *
import sys, os

stdinQueue = OnRequestQueue()

def readNextInput():
	ch = os.read(sys.stdin.fileno(),7)
	stdinQueue.put(ch)

oldTermios = None

def setTtyNoncanonical(fd, timeout=0):
	global oldTermios
	import termios
	old = termios.tcgetattr(fd)
	if not oldTermios: oldTermios = old
	new = termios.tcgetattr(fd)
	new[3] = new[3] & ~termios.ICANON & ~termios.ECHO
	# http://www.unixguide.net/unix/programming/3.6.2.shtml
	#new[6] [termios.VMIN] = 1
	#new[6] [termios.VTIME] = 0
	new[6] [termios.VMIN] = 0 if timeout > 0 else 1
	timeout *= 10 # 10ths of second
	if timeout > 0 and timeout < 1: timeout = 1
	new[6] [termios.VTIME] = timeout

	termios.tcsetattr(fd, termios.TCSANOW, new)
	termios.tcsendbreak(fd,0)

def restoreTty(fd):
	if oldTermios:
		import termios
		termios.tcsetattr(fd, termios.TCSANOW, oldTermios)
		termios.tcsendbreak(fd, 0)

from State import state, reloadModules

def printState():
	if state.curSong:
		try:
			print os.path.basename(state.curSong.url) + " : " + \
				formatTime(state.player.curSongPos) + " / " + \
				formatTime(state.player.curSongLen)
		except:
			print "song ???"
	else:
		print "no song"

def handleInput(ch):
	if ch == "q" or ch == "\0" or ch == "":
		print "stdin: quit"
		state.quit()
	try:
		if ch == "\x1b[D": # left
			state.player.seekRel(-10)
			printState()
		elif ch == "\x1b[C": #right
			state.player.seekRel(10)
			printState()
		elif ch == "\n": # return
			state.nextSong()
		elif ch == " ":
			state.playPause()
		elif ch == "r":
			reloadModules()
	except:
		sys.excepthook(*sys.exc_info())

def stdinconsoleMain():
	import appinfo
	if appinfo.args.shell:
		print "Setting up Python shell interface"
		# This is hacky for now but actually I don't really know a better way.
		# The problem is that the main thread might be used by the GUI.
		# Thus, we can only cancel this here via the shell itself.
		from better_exchook import debug_shell
		debug_shell({},{"state":state})
		# The shell exited. Quit.
		state.quit()
		return

	fd = sys.stdin.fileno()
	if not os.isatty(fd): return # don't do anything

	try:
		# This import will fail if there is no GUI.
		import gui

	except: # no GUI

		# the main thread is pushing stdin updates to stdinQueue.
		setTtyNoncanonical(sys.stdin.fileno())
		print "stdin input ready"

		for ch in stdinQueue.read():
			handleInput(ch)

	else:
		# the GUI is handling the main thread.
		# it means we dont get stdinQueue updates here.
		# as we need a way to be able to cancel this,
		# we only can frequently check with a timeout here.
		setTtyNoncanonical(fd, timeout=1)

		from threading import currentThread
		thread = currentThread()
		print "stdin input ready"
		while not thread.cancel:
			ch = os.read(fd,7)
			if ch:
				handleInput(ch)

	restoreTty(fd)
<commit_msg>shell: use all globals()<commit_after>
from utils import *
import sys, os

stdinQueue = OnRequestQueue()

def readNextInput():
	ch = os.read(sys.stdin.fileno(),7)
	stdinQueue.put(ch)

oldTermios = None

def setTtyNoncanonical(fd, timeout=0):
	global oldTermios
	import termios
	old = termios.tcgetattr(fd)
	if not oldTermios: oldTermios = old
	new = termios.tcgetattr(fd)
	new[3] = new[3] & ~termios.ICANON & ~termios.ECHO
	# http://www.unixguide.net/unix/programming/3.6.2.shtml
	#new[6] [termios.VMIN] = 1
	#new[6] [termios.VTIME] = 0
	new[6] [termios.VMIN] = 0 if timeout > 0 else 1
	timeout *= 10 # 10ths of second
	if timeout > 0 and timeout < 1: timeout = 1
	new[6] [termios.VTIME] = timeout

	termios.tcsetattr(fd, termios.TCSANOW, new)
	termios.tcsendbreak(fd,0)

def restoreTty(fd):
	if oldTermios:
		import termios
		termios.tcsetattr(fd, termios.TCSANOW, oldTermios)
		termios.tcsendbreak(fd, 0)

from State import state, reloadModules

def printState():
	if state.curSong:
		try:
			print os.path.basename(state.curSong.url) + " : " + \
				formatTime(state.player.curSongPos) + " / " + \
				formatTime(state.player.curSongLen)
		except:
			print "song ???"
	else:
		print "no song"

def handleInput(ch):
	if ch == "q" or ch == "\0" or ch == "":
		print "stdin: quit"
		state.quit()
	try:
		if ch == "\x1b[D": # left
			state.player.seekRel(-10)
			printState()
		elif ch == "\x1b[C": #right
			state.player.seekRel(10)
			printState()
		elif ch == "\n": # return
			state.nextSong()
		elif ch == " ":
			state.playPause()
		elif ch == "r":
			reloadModules()
	except:
		sys.excepthook(*sys.exc_info())

def stdinconsoleMain():
	import appinfo
	if appinfo.args.shell:
		print "Setting up Python shell interface"
		# This is hacky for now but actually I don't really know a better way.
		# The problem is that the main thread might be used by the GUI.
		# Thus, we can only cancel this here via the shell itself.
		from better_exchook import debug_shell
		debug_shell({},globals())
		# The shell exited. Quit.
		state.quit()
		return

	fd = sys.stdin.fileno()
	if not os.isatty(fd): return # don't do anything

	try:
		# This import will fail if there is no GUI.
		import gui

	except: # no GUI

		# the main thread is pushing stdin updates to stdinQueue.
		setTtyNoncanonical(sys.stdin.fileno())
		print "stdin input ready"

		for ch in stdinQueue.read():
			handleInput(ch)

	else:
		# the GUI is handling the main thread.
		# it means we dont get stdinQueue updates here.
		# as we need a way to be able to cancel this,
		# we only can frequently check with a timeout here.
		setTtyNoncanonical(fd, timeout=1)

		from threading import currentThread
		thread = currentThread()
		print "stdin input ready"
		while not thread.cancel:
			ch = os.read(fd,7)
			if ch:
				handleInput(ch)

	restoreTty(fd)
<|endoftext|>


<commit_before># inspired from:

# https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
# https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py

__name__ = 'thinc'
__version__ = '6.4.0'
__summary__ = "Practical Machine Learning for NLP"
__uri__ = 'https://github.com/explosion/thinc'
__author__ = 'Matthew Honnibal'
__email__ = 'matt@explosion.ai'
__license__ = 'MIT'
__title__ = "thinc"
__release__ = False
<commit_msg>Update release<commit_after># inspired from:

# https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
# https://github.com/pypa/warehouse/blob/master/warehouse/__about__.py

__name__ = 'thinc'
__version__ = '6.4.0'
__summary__ = "Practical Machine Learning for NLP"
__uri__ = 'https://github.com/explosion/thinc'
__author__ = 'Matthew Honnibal'
__email__ = 'matt@explosion.ai'
__license__ = 'MIT'
__title__ = "thinc"
__release__ = True
<|endoftext|>


<commit_before>"""
WebAPI provides a thin gevent cooperative wrapper over Steam's WebAPI

Calling an endpoint

.. code:: python

    >>> api = WebAPI(key)
    >>> api.ISteamUser.ResolveVanityURL(vanityurl="valve", url_type=2)
    >>> api.call('ISteamUser.ResolveVanityURL', vanityurl="valve", url_type=2)
    {u'response': {u'steamid': u'103582791429521412', u'success': 1}}

All globals params (``key``, ``https``, ``format``, ``raw``) can be specified on per call basis.

.. code:: python

    >>> print a.ISteamUser.ResolveVanityURL(format='vdf', raw=True, vanityurl="valve", url_type=2)
    "response"
    {
            "steamid"       "103582791429521412"
            "success"       "1"
    }
"""
from steam.util.web import make_requests_session

DEFAULT_PARAMS = {
    # api parameters
    'key': None,
    'format': 'json',
    # internal
    'https': True,
    'http_timeout': 30,
    'raw': False,
}

def webapi_request(path, method='GET', caller=None, params={}, session=None):
    """
    Low level function for calling Steam's WebAPI

    :param path: request url
    :type path: :class:`str`
    :param method: HTTP method (GET or POST)
    :type method: :class:`str`
    :param caller: caller reference, caller.last_response is set to the last response
    :param params: dict of WebAPI and endpoint specific params
    :type params: :class:`dict`
    :param session: an instance requests session, or one is created per call
    :type session: :class:`requests.Session`
    :return: response based on paramers
    :rtype: :class:`dict`, :class:`lxml.etree.Element`, :class:`str`
    """
    if method not in ('GET', 'POST'):
        raise NotImplemented("HTTP method: %s" % repr(self.method))

    onetime = {}
    for param in DEFAULT_PARAMS:
        params[param] = onetime[param] = params.get(param,
                                                    DEFAULT_PARAMS[param],
                                                    )
    path = "%s://api.steampowered.com/%s" % ('https' if params.get('https', True) else 'http',
                                             path)
    del params['raw']
    del params['https']
    del params['http_timeout']

    if onetime['format'] not in ('json', 'vdf', 'xml'):
        raise ValueError("Expected format to be json,vdf or xml; got %s" % onetime['format'])

    # move params to data, if data is not specified for POST
    # simplifies code calling this method
    kwargs = {'params': params} if method == "GET" else {'data': params}

    f = getattr(session, method.lower())
    resp = f(path, stream=False, timeout=onetime['http_timeout'], **kwargs)

    # we keep a reference of the last response instance on the caller
    if caller is not None:
        caller.last_response = resp

    # 4XX and 5XX will cause this to raise
    resp.raise_for_status()

    # response
    if onetime['raw']:
        return resp.text

    if onetime['format'] == 'json':
        return resp.json()
    elif onetime['format'] == 'xml':
        import lxml.etree
        return lxml.etree.fromstring(resp.content)
    elif onetime['format'] == 'vdf':
        import vdf
        return vdf.loads(resp.text)

class WebAPI(object):
    """
    Steam WebAPI wrapper. See https://developer.valvesoftware.com/wiki/Steam_Web_API

    .. note::
        Interfaces and methods are populated automatically from WebAPI.

    :param key: api key from https://steamcommunity.com/dev/apikey
    :type key: :class:`str`
    :param format: response format, either (``json``, ``vdf``, or ``xml``) only when ``raw=False``
    :type format: :class:`str`
    :param raw: return raw response
    :type raw: class:`bool`
    :param https: use ``https``
    :type https: :class:`bool`
    :param http_timeout: HTTP timeout in seconds
    :type http_timeout: :class:`int`
    :param auto_load_interfaces: load interfaces from the WebAPI
    :type auto_load_interfaces: :class:`bool`

    These can be specified per method call for one off calls
    """
    key = DEFAULT_PARAMS['key']
    format = DEFAULT_PARAMS['format']
    raw = DEFAULT_PARAMS['raw']
    https = DEFAULT_PARAMS['https']
    http_timeout = DEFAULT_PARAMS['http_timeout']
    interfaces = []

    def __init__(self, key, format = DEFAULT_PARAMS['format'],
                            raw = DEFAULT_PARAMS['raw'],
                            https = DEFAULT_PARAMS['https'],
                            http_timeout = DEFAULT_PARAMS['http_timeout'],
                            auto_load_interfaces = True):
        self.key = key                              #: api key
        self.format = format                        #: format (``json``, ``vdf``, or ``xml``)
        self.raw = raw                              #: return raw reponse or parse
        self.https = https                          #: use https or not
        self.http_timeout = http_timeout            #: HTTP timeout in seconds
        self.interfaces = []                        #: list of all interfaces
        self.session = make_requests_session()      #: :class:`requests.Session` from :func:`steam.util.web.make_requests_session`

        if auto_load_interfaces:
            self.load_interfaces(self.fetch_interfaces())

    def __repr__(self):
        return "%s(key=%s, https=%s)" % (
            self.__class__.__name__,
            repr(self.key),
            repr(self.https),
            )

    def fetch_interfaces(self):
        """
        Returns a dict with the response from ``GetSupportedAPIList``

        :return: :class:`dict` of all interfaces and methods

        The returned value can passed to :py:func:`WebAPI.load_interfaces`
        """
        return webapi_request(
            "ISteamWebAPIUtil/GetSupportedAPIList/v1/",
            method="GET",
            caller=None,
            params={'format': 'json',
                    'key': self.key,
                    },
            session=self.session,
            )

    def load_interfaces(self, interfaces_dict):
        """
        Populates the namespace under the instance
        """
        if interfaces_dict.get('apilist', {}).get('interfaces', None) is None:
            raise ValueError("Invalid response for GetSupportedAPIList")

        interfaces = interfaces_dict['apilist']['interfaces']
        if len(interfaces) == 0:
            raise ValueError("API returned not interfaces; probably using invalid key")

        # clear existing interface instances
        for interface in self.interfaces:
            delattr(self, interface.name)
        self.interfaces = []

        # create interface instances from response
        for interface in interfaces:
            obj = WebAPIInterface(interface, parent=self)
            self.interfaces.append(obj)
            setattr(self, obj.name, obj)

    def call(self, method_path, **kwargs):
        """
        Make an API call for specific method

        :param method_path: format ``Interface.Method`` (e.g. ``ISteamWebAPIUtil.GetServerInfo``)
        :type method_path: :class:`str`
        :param kwargs: keyword arguments for the specific method
        :return: response
        :rtype: :class:`dict`, :class:`lxml.etree.Element` or :class:`str`
        """

        interface, method = method_path.split('.', 1)
        return getattr(getattr(self, interface), method)(**kwargs)

    def doc(self):
        """
        Print documentation for all interfaces and their methods
        """

        doc = "Steam Web API - List of all interfaces\n\n"
        for interface in self.interfaces:
            doc += interface.__doc__
        print(doc)

class WebAPIInterface(object):
    """
    Steam Web API Interface
    """

    def __init__(self, interface_dict, parent):
        self._parent = parent
        self.name = interface_dict['name']
        self.methods = []

        for method in interface_dict['methods']:
            obj = WebAPIMethod(method, parent=self)
            self.methods.append(obj)

            # map the method name as attribute including version
            setattr(self, "%s_v%d" % (obj.name, obj.version), obj)

            # without version, but th refernce of latest version
            current_obj = getattr(self, obj.name, None)
            if current_obj is None or current_obj.version < obj.version:
                setattr(self, obj.name, obj)

    def __repr__(self):
        return "<%s %s with %s methods>" % (
            self.__class__.__name__,
            repr(self.name),
            repr(len(list(self))),
            )

    def __iter__(self):
        return iter(self.methods)

    @property
    def key(self):
        return self._parent.key

    @property
    def https(self):
        return self._parent.https

    @property
    def http_timeout(self):
        return self._parent.http_timeout

    @property
    def format(self):
        return self._parent.format

    @property
    def raw(self):
        return self._parent.raw

    @property
    def session(self):
        return self._parent.session

    def doc(self):
        print(self.__doc__)

    @property
    def __doc__(self):
        doc = "%s\n%s\n" % (self.name, '-'*len(self.name))
        for method in self.methods:
            doc += "  %s\n" % method.__doc__.replace("\n", "\n  ")
        return doc

class WebAPIMethod(object):
    """
    Steam Web API Interface Method
    """

    def __init__(self, method_dict, parent):
        self.last_response = None
        self._parent = parent
        self._dict = method_dict

        params = method_dict['parameters']
        self._dict['parameters'] = {}
        for param in params:
            # add property indicating param can be a list
            param['_array'] = param['name'].endswith('[0]')
            # remove array suffix
            if param['_array']:
                param['name'] = param['name'][:-3]
            # turn params from a list to a dict
            self._dict['parameters'][param['name']] = param

    def __repr__(self):
        return "<%s %s>" % (
            self.__class__.__name__,
            repr("%s.%s_v%d" % (
                self._parent.name,
                self.name,
                self.version,
                )),
            )

    def __call__(self, **kwargs):
        possible_kwargs = set(self._dict['parameters'].keys()) | set(DEFAULT_PARAMS.keys())
        unrecognized = set(kwargs.keys()).difference(possible_kwargs)
        if unrecognized:
            raise ValueError("Unrecognized parameter %s" % repr(unrecognized.pop()))

        params = {}
        # process special case kwargs
        for param in DEFAULT_PARAMS.keys():
            if param in kwargs:
                params[param] = kwargs[param]
                del kwargs[param]
            else:
                params[param] = getattr(self._parent, param)

        # process method parameters
        for param in self.parameters.values():
            name = param['name']
            islist = param['_array']
            optional = param['optional']

            # raise if we are missing a required parameter
            if not optional and name not in kwargs and name != 'key':
                raise ValueError("Method requires %s to be set" % repr(name))

            # populate params that will be passed to _api_request
            if name in kwargs:
                # some parameters can be an array, they need to be send as seperate field
                # the array index is append to the name (e.g. name[0], name[1] etc)
                if islist:
                    if not isinstance(kwargs[name], list):
                        raise ValueError("Expected %s to be a list, got %s" % (
                            repr(name),
                            repr(type(kwargs[name])))
                            )

                    for idx, value in enumerate(kwargs[name]):
                        params['%s[%d]' % (name, idx)] = value
                else:
                    params[name] = kwargs[name]

        # make the request
        return webapi_request(
            "%s/%s/v%s/" % (self._parent.name, self.name, self.version),
            method=self.method,
            caller=self,
            params=params,
            session=self._parent.session,
            )

    @property
    def version(self):
        return self._dict['version']

    @property
    def method(self):
        return self._dict['httpmethod']

    @property
    def parameters(self):
        return self._dict['parameters']

    @property
    def name(self):
        return self._dict['name']

    def doc(self):
        print(self.__doc__)

    @property
    def __doc__(self):
        doc = "%(httpmethod)s %(name)s (v%(version)04d)\n" % self._dict

        if 'description' in self._dict:
            doc += "\n  %(description)s\n" % self._dict

        if len(self.parameters):
            doc += "  \n  Parameters:\n"

            for param in sorted(self.parameters.values(), key=lambda x: x['name']):
                doc += "    %s %s %s%s\n" % (
                    param['name'].ljust(25),
                    ((param['type']+"[]") if param['_array'] else
                     param['type']
                     ).ljust(8),
                    'optional' if param['optional'] else 'required',
                    (("\n      - " + param['description'])
                     if 'description' in param and param['description'] else ''
                     ),
                    )

        return doc
<commit_msg>WebAPI: fix param type serialization<commit_after>"""
WebAPI provides a thin gevent cooperative wrapper over Steam's WebAPI

Calling an endpoint

.. code:: python

    >>> api = WebAPI(key)
    >>> api.ISteamUser.ResolveVanityURL(vanityurl="valve", url_type=2)
    >>> api.call('ISteamUser.ResolveVanityURL', vanityurl="valve", url_type=2)
    {u'response': {u'steamid': u'103582791429521412', u'success': 1}}

All globals params (``key``, ``https``, ``format``, ``raw``) can be specified on per call basis.

.. code:: python

    >>> print a.ISteamUser.ResolveVanityURL(format='vdf', raw=True, vanityurl="valve", url_type=2)
    "response"
    {
            "steamid"       "103582791429521412"
            "success"       "1"
    }
"""
import json
from steam.util.web import make_requests_session

DEFAULT_PARAMS = {
    # api parameters
    'key': None,
    'format': 'json',
    # internal
    'https': True,
    'http_timeout': 30,
    'raw': False,
}

def webapi_request(path, method='GET', caller=None, params={}, session=None):
    """
    Low level function for calling Steam's WebAPI

    :param path: request url
    :type path: :class:`str`
    :param method: HTTP method (GET or POST)
    :type method: :class:`str`
    :param caller: caller reference, caller.last_response is set to the last response
    :param params: dict of WebAPI and endpoint specific params
    :type params: :class:`dict`
    :param session: an instance requests session, or one is created per call
    :type session: :class:`requests.Session`
    :return: response based on paramers
    :rtype: :class:`dict`, :class:`lxml.etree.Element`, :class:`str`
    """
    if method not in ('GET', 'POST'):
        raise NotImplemented("HTTP method: %s" % repr(self.method))

    onetime = {}
    for param in DEFAULT_PARAMS:
        params[param] = onetime[param] = params.get(param,
                                                    DEFAULT_PARAMS[param],
                                                    )
    path = "%s://api.steampowered.com/%s" % ('https' if params.get('https', True) else 'http',
                                             path)
    del params['raw']
    del params['https']
    del params['http_timeout']

    if onetime['format'] not in ('json', 'vdf', 'xml'):
        raise ValueError("Expected format to be json,vdf or xml; got %s" % onetime['format'])

    # serialize some parameter types properly
    for k, v in params.items():
        if isinstance(v, bool):
            params[k] = 1 if v else 0
        elif isinstance(v, (list, dict)):
            params[k] = json.dumps(v)

    # move params to data, if data is not specified for POST
    # simplifies code calling this method
    kwargs = {'params': params} if method == "GET" else {'data': params}

    f = getattr(session, method.lower())
    resp = f(path, stream=False, timeout=onetime['http_timeout'], **kwargs)

    # we keep a reference of the last response instance on the caller
    if caller is not None:
        caller.last_response = resp

    # 4XX and 5XX will cause this to raise
    resp.raise_for_status()

    # response
    if onetime['raw']:
        return resp.text

    if onetime['format'] == 'json':
        return resp.json()
    elif onetime['format'] == 'xml':
        import lxml.etree
        return lxml.etree.fromstring(resp.content)
    elif onetime['format'] == 'vdf':
        import vdf
        return vdf.loads(resp.text)

class WebAPI(object):
    """
    Steam WebAPI wrapper. See https://developer.valvesoftware.com/wiki/Steam_Web_API

    .. note::
        Interfaces and methods are populated automatically from WebAPI.

    :param key: api key from https://steamcommunity.com/dev/apikey
    :type key: :class:`str`
    :param format: response format, either (``json``, ``vdf``, or ``xml``) only when ``raw=False``
    :type format: :class:`str`
    :param raw: return raw response
    :type raw: class:`bool`
    :param https: use ``https``
    :type https: :class:`bool`
    :param http_timeout: HTTP timeout in seconds
    :type http_timeout: :class:`int`
    :param auto_load_interfaces: load interfaces from the WebAPI
    :type auto_load_interfaces: :class:`bool`

    These can be specified per method call for one off calls
    """
    key = DEFAULT_PARAMS['key']
    format = DEFAULT_PARAMS['format']
    raw = DEFAULT_PARAMS['raw']
    https = DEFAULT_PARAMS['https']
    http_timeout = DEFAULT_PARAMS['http_timeout']
    interfaces = []

    def __init__(self, key, format = DEFAULT_PARAMS['format'],
                            raw = DEFAULT_PARAMS['raw'],
                            https = DEFAULT_PARAMS['https'],
                            http_timeout = DEFAULT_PARAMS['http_timeout'],
                            auto_load_interfaces = True):
        self.key = key                              #: api key
        self.format = format                        #: format (``json``, ``vdf``, or ``xml``)
        self.raw = raw                              #: return raw reponse or parse
        self.https = https                          #: use https or not
        self.http_timeout = http_timeout            #: HTTP timeout in seconds
        self.interfaces = []                        #: list of all interfaces
        self.session = make_requests_session()      #: :class:`requests.Session` from :func:`steam.util.web.make_requests_session`

        if auto_load_interfaces:
            self.load_interfaces(self.fetch_interfaces())

    def __repr__(self):
        return "%s(key=%s, https=%s)" % (
            self.__class__.__name__,
            repr(self.key),
            repr(self.https),
            )

    def fetch_interfaces(self):
        """
        Returns a dict with the response from ``GetSupportedAPIList``

        :return: :class:`dict` of all interfaces and methods

        The returned value can passed to :py:func:`WebAPI.load_interfaces`
        """
        return webapi_request(
            "ISteamWebAPIUtil/GetSupportedAPIList/v1/",
            method="GET",
            caller=None,
            params={'format': 'json',
                    'key': self.key,
                    },
            session=self.session,
            )

    def load_interfaces(self, interfaces_dict):
        """
        Populates the namespace under the instance
        """
        if interfaces_dict.get('apilist', {}).get('interfaces', None) is None:
            raise ValueError("Invalid response for GetSupportedAPIList")

        interfaces = interfaces_dict['apilist']['interfaces']
        if len(interfaces) == 0:
            raise ValueError("API returned not interfaces; probably using invalid key")

        # clear existing interface instances
        for interface in self.interfaces:
            delattr(self, interface.name)
        self.interfaces = []

        # create interface instances from response
        for interface in interfaces:
            obj = WebAPIInterface(interface, parent=self)
            self.interfaces.append(obj)
            setattr(self, obj.name, obj)

    def call(self, method_path, **kwargs):
        """
        Make an API call for specific method

        :param method_path: format ``Interface.Method`` (e.g. ``ISteamWebAPIUtil.GetServerInfo``)
        :type method_path: :class:`str`
        :param kwargs: keyword arguments for the specific method
        :return: response
        :rtype: :class:`dict`, :class:`lxml.etree.Element` or :class:`str`
        """

        interface, method = method_path.split('.', 1)
        return getattr(getattr(self, interface), method)(**kwargs)

    def doc(self):
        """
        Print documentation for all interfaces and their methods
        """

        doc = "Steam Web API - List of all interfaces\n\n"
        for interface in self.interfaces:
            doc += interface.__doc__
        print(doc)

class WebAPIInterface(object):
    """
    Steam Web API Interface
    """

    def __init__(self, interface_dict, parent):
        self._parent = parent
        self.name = interface_dict['name']
        self.methods = []

        for method in interface_dict['methods']:
            obj = WebAPIMethod(method, parent=self)
            self.methods.append(obj)

            # map the method name as attribute including version
            setattr(self, "%s_v%d" % (obj.name, obj.version), obj)

            # without version, but th refernce of latest version
            current_obj = getattr(self, obj.name, None)
            if current_obj is None or current_obj.version < obj.version:
                setattr(self, obj.name, obj)

    def __repr__(self):
        return "<%s %s with %s methods>" % (
            self.__class__.__name__,
            repr(self.name),
            repr(len(list(self))),
            )

    def __iter__(self):
        return iter(self.methods)

    @property
    def key(self):
        return self._parent.key

    @property
    def https(self):
        return self._parent.https

    @property
    def http_timeout(self):
        return self._parent.http_timeout

    @property
    def format(self):
        return self._parent.format

    @property
    def raw(self):
        return self._parent.raw

    @property
    def session(self):
        return self._parent.session

    def doc(self):
        print(self.__doc__)

    @property
    def __doc__(self):
        doc = "%s\n%s\n" % (self.name, '-'*len(self.name))
        for method in self.methods:
            doc += "  %s\n" % method.__doc__.replace("\n", "\n  ")
        return doc

class WebAPIMethod(object):
    """
    Steam Web API Interface Method
    """

    def __init__(self, method_dict, parent):
        self.last_response = None
        self._parent = parent
        self._dict = method_dict

        params = method_dict['parameters']
        self._dict['parameters'] = {}
        for param in params:
            # add property indicating param can be a list
            param['_array'] = param['name'].endswith('[0]')
            # remove array suffix
            if param['_array']:
                param['name'] = param['name'][:-3]
            # turn params from a list to a dict
            self._dict['parameters'][param['name']] = param

    def __repr__(self):
        return "<%s %s>" % (
            self.__class__.__name__,
            repr("%s.%s_v%d" % (
                self._parent.name,
                self.name,
                self.version,
                )),
            )

    def __call__(self, **kwargs):
        possible_kwargs = set(self._dict['parameters'].keys()) | set(DEFAULT_PARAMS.keys())
        unrecognized = set(kwargs.keys()).difference(possible_kwargs)
        if unrecognized:
            raise ValueError("Unrecognized parameter %s" % repr(unrecognized.pop()))

        params = {}
        # process special case kwargs
        for param in DEFAULT_PARAMS.keys():
            if param in kwargs:
                params[param] = kwargs[param]
                del kwargs[param]
            else:
                params[param] = getattr(self._parent, param)

        # process method parameters
        for param in self.parameters.values():
            name = param['name']
            islist = param['_array']
            optional = param['optional']

            # raise if we are missing a required parameter
            if not optional and name not in kwargs and name != 'key':
                raise ValueError("Method requires %s to be set" % repr(name))

            # populate params that will be passed to _api_request
            if name in kwargs:
                # some parameters can be an array, they need to be send as seperate field
                # the array index is append to the name (e.g. name[0], name[1] etc)
                if islist:
                    if not isinstance(kwargs[name], list):
                        raise ValueError("Expected %s to be a list, got %s" % (
                            repr(name),
                            repr(type(kwargs[name])))
                            )

                    for idx, value in enumerate(kwargs[name]):
                        params['%s[%d]' % (name, idx)] = value
                else:
                    params[name] = kwargs[name]

        # make the request
        return webapi_request(
            "%s/%s/v%s/" % (self._parent.name, self.name, self.version),
            method=self.method,
            caller=self,
            params=params,
            session=self._parent.session,
            )

    @property
    def version(self):
        return self._dict['version']

    @property
    def method(self):
        return self._dict['httpmethod']

    @property
    def parameters(self):
        return self._dict['parameters']

    @property
    def name(self):
        return self._dict['name']

    def doc(self):
        print(self.__doc__)

    @property
    def __doc__(self):
        doc = "%(httpmethod)s %(name)s (v%(version)04d)\n" % self._dict

        if 'description' in self._dict:
            doc += "\n  %(description)s\n" % self._dict

        if len(self.parameters):
            doc += "  \n  Parameters:\n"

            for param in sorted(self.parameters.values(), key=lambda x: x['name']):
                doc += "    %s %s %s%s\n" % (
                    param['name'].ljust(25),
                    ((param['type']+"[]") if param['_array'] else
                     param['type']
                     ).ljust(8),
                    'optional' if param['optional'] else 'required',
                    (("\n      - " + param['description'])
                     if 'description' in param and param['description'] else ''
                     ),
                    )

        return doc
<|endoftext|>