<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import default
import requests
from terminaltables import AsciiTable
from spider163.utils import pylog

offset = 0
limit = 20
type = {1: "歌曲", 10: "专辑", 100: "歌手", 1000: "歌单"}

def searchSong(key):
    url = default.search_api
    data = {'s': key, 'offset': 0, 'limit': 20, 'type': "1"}
    req = requests.post(url, headers=default.header, data=data, timeout=10)
    songs = req.json()["result"]['songs']
    song_table = AsciiTable([["ID", "歌曲", "专辑", "演唱"]])
    for item in songs:
        id = item['id']
        name = (item['name']).encode("utf-8")
        album = (item['album']['name']).encode("utf-8")
        artist = []
        for a in item['artists']:
            artist.append(a['name'].encode("utf-8"))
        song_table.table_data.append([str(id), name, album, ",".join(artist)])
    print(pylog.Blue("与 \"{}\" 有关的歌曲".format(key)))
    print(song_table.table)

def searchAlbum(key):
    url = default.search_api
    data = {'s': key, 'offset': 0, 'limit': 20, 'type': "10"}
    req = requests.post(url, headers=default.header, data=data, timeout=10)
    albums = req.json()["result"]['albums']
    song_table = AsciiTable([["ID", "专辑", "演唱","发行方"]])
    for item in albums:
        id = item['id']
        name = item['name'].encode("utf-8")
        company = ""
        if item['company'] !=  None:
            company = item['company'].encode("utf-8")
        artist = []
        for a in item['artists']:
            artist.append(a['name'].encode("utf-8"))
        song_table.table_data.append([str(id), name, ",".join(artist), company])
    print(pylog.Blue("与 \"{}\" 有关的专辑".format(key)))
    print(song_table.table)

def searchSinger(key):
    url = default.search_api
    data = {'s': key, 'offset': 0, 'limit': 10, 'type': "100"}
    req = requests.post(url, headers=default.header, data=data, timeout=10)
    artists = req.json()["result"]['artists']
    song_table = AsciiTable([["ID", "姓名", "专辑数量", "MV数量"]])
    for item in artists:
        id = str(item['id'])
        name = item['name'].encode("utf-8")
        acount = str(item['albumSize'])
        mcount = str(item['mvSize'])
        song_table.table_data.append([id, name, acount, mcount])
    print(pylog.Blue("与 \"{}\" 有关的歌手".format(key)))
    print(song_table.table)

def searchPlaylist(key):
    url = default.search_api
    data = {'s': key, 'offset': 0, 'limit': 5, 'type': "1000"}
    req = requests.post(url, headers=default.header, data=data, timeout=10)
    playlists = req.json()["result"]['playlists']
    song_table = AsciiTable([["ID", "歌单", "维护者", "播放数量", "收藏数量"]])
    for item in playlists:
        id = str(item['id'])
        name = item['name'].encode("utf-8")
        creator = item['creator']['nickname'].encode("utf-8")
        pcount = str(item['playCount'])
        bcount = str(item['bookCount'])
        song_table.table_data.append([id, name, creator, pcount, bcount])
    print(pylog.Blue("与 \"{}\" 有关的歌单".format(key)))
    print(song_table.table)

if __name__ == "__main__":
    searchSong("林依晨")<commit_msg>fix:增加搜索无结果的异常处理<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import default
import requests
from terminaltables import AsciiTable
from spider163.utils import pylog

offset = 0
limit = 20
type = {1: "歌曲", 10: "专辑", 100: "歌手", 1000: "歌单"}

def searchSong(key):
    url = default.search_api
    data = {'s': key, 'offset': 0, 'limit': 20, 'type': "1"}
    req = requests.post(url, headers=default.header, data=data, timeout=10)
    if req.json()["result"]['songCount'] == 0:
        pylog.log.warn("关键词 {} 没有可搜索歌曲".format(key))
        return
    songs = req.json()["result"]['songs']
    song_table = AsciiTable([["ID", "歌曲", "专辑", "演唱"]])
    for item in songs:
        id = item['id']
        name = (item['name']).encode("utf-8")
        album = (item['album']['name']).encode("utf-8")
        artist = []
        for a in item['artists']:
            artist.append(a['name'].encode("utf-8"))
        song_table.table_data.append([str(id), name, album, ",".join(artist)])
    print(pylog.Blue("与 \"{}\" 有关的歌曲".format(key)))
    print(song_table.table)

def searchAlbum(key):
    url = default.search_api
    data = {'s': key, 'offset': 0, 'limit': 20, 'type': "10"}
    req = requests.post(url, headers=default.header, data=data, timeout=10)
    if req.json()["result"]['albumCount'] == 0:
        pylog.log.warn("关键词 {} 没有可搜索专辑".format(key))
        return
    albums = req.json()["result"]['albums']
    song_table = AsciiTable([["ID", "专辑", "演唱","发行方"]])
    for item in albums:
        id = item['id']
        name = item['name'].encode("utf-8")
        company = ""
        if item['company'] !=  None:
            company = item['company'].encode("utf-8")
        artist = []
        for a in item['artists']:
            artist.append(a['name'].encode("utf-8"))
        song_table.table_data.append([str(id), name, ",".join(artist), company])
    print(pylog.Blue("与 \"{}\" 有关的专辑".format(key)))
    print(song_table.table)

def searchSinger(key):
    url = default.search_api
    data = {'s': key, 'offset': 0, 'limit': 10, 'type': "100"}
    req = requests.post(url, headers=default.header, data=data, timeout=10)
    if req.json()["result"]['artistCount'] == 0:
        pylog.log.warn("关键词 {} 没有可搜索艺术家".format(key))
        return
    artists = req.json()["result"]['artists']
    song_table = AsciiTable([["ID", "姓名", "专辑数量", "MV数量"]])
    for item in artists:
        id = str(item['id'])
        name = item['name'].encode("utf-8")
        acount = str(item['albumSize'])
        mcount = str(item['mvSize'])
        song_table.table_data.append([id, name, acount, mcount])
    print(pylog.Blue("与 \"{}\" 有关的歌手".format(key)))
    print(song_table.table)

def searchPlaylist(key):
    url = default.search_api
    data = {'s': key, 'offset': 0, 'limit': 5, 'type': "1000"}
    req = requests.post(url, headers=default.header, data=data, timeout=10)
    if req.json()["result"]['playlistCount'] == 0:
        pylog.log.warn("关键词 {} 没有可搜索歌单".format(key))
        return
    playlists = req.json()["result"]['playlists']
    song_table = AsciiTable([["ID", "歌单", "维护者", "播放数量", "收藏数量"]])
    for item in playlists:
        id = str(item['id'])
        name = item['name'].encode("utf-8")
        creator = item['creator']['nickname'].encode("utf-8")
        pcount = str(item['playCount'])
        bcount = str(item['bookCount'])
        song_table.table_data.append([id, name, creator, pcount, bcount])
    print(pylog.Blue("与 \"{}\" 有关的歌单".format(key)))
    print(song_table.table)

if __name__ == "__main__":
    searchSong("林依晨")<|endoftext|>


<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from libthumbor import CryptoURL

class ThumborService(object):

    def __init__(self, baseurl='http://localhost:8888/', secretkey='MY_SECURE_KEY'):
        self.baseurl = baseurl.rstrip('/')
        self._urlgen = CryptoURL(key=secretkey)

    def generate_url(self, options):
        return self.baseurl + self._urlgen.generate(**options)

class ThumborUrlGenerator(object):

    def __init__(self, service, key='content_url', destkey='thumbnail', options=None, optionskey=None):
        self.service = service
        self.key = key
        self.destkey = destkey
        self.options = options
        self.optionskey = optionskey

        if not (self.options is None) ^ (self.optionskey is None):
            raise ValueError('Either options or optionskey is required')

    def __call__(self, item, send):
        for oid in item['inserts']:
            if self.options:
                options = self.options.copy()
            else:
                options = item['data'][oid][self.optionskey].copy()

            options['image_url'] = item['data'][oid][self.key]
            item['data'][oid][self.destkey] = self.service.generate_url(options)

        send(item, self)

    @property
    def dependencies(self):
        yield (self, self.service)
<commit_msg>Remove dependencies property<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

from libthumbor import CryptoURL

class ThumborService(object):

    def __init__(self, baseurl='http://localhost:8888/', secretkey='MY_SECURE_KEY'):
        self.baseurl = baseurl.rstrip('/')
        self._urlgen = CryptoURL(key=secretkey)

    def generate_url(self, options):
        return self.baseurl + self._urlgen.generate(**options)

class ThumborUrlGenerator(object):

    def __init__(self, service, key='content_url', destkey='thumbnail', options=None, optionskey=None):
        self.service = service
        self.key = key
        self.destkey = destkey
        self.options = options
        self.optionskey = optionskey

        if not (self.options is None) ^ (self.optionskey is None):
            raise ValueError('Either options or optionskey is required')

    def __call__(self, item, send):
        for oid in item['inserts']:
            if self.options:
                options = self.options.copy()
            else:
                options = item['data'][oid][self.optionskey].copy()

            options['image_url'] = item['data'][oid][self.key]
            item['data'][oid][self.destkey] = self.service.generate_url(options)

        send(item, self)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright (C) 1999-2015, Raffaele Salmaso <raffaele@salmaso.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import absolute_import, division, print_function, unicode_literals
from stua.version import get_version, HG

VERSION = (1, 0, 0, "final", 0)

__author__ = "Raffaele Salmaso"
__author_email__ = "raffaele@salmaso.org"
__version__ = get_version(VERSION, HG, __file__)
<commit_msg>update version to 1.0.1<commit_after># -*- coding: utf-8 -*-

# Copyright (C) 1999-2015, Raffaele Salmaso <raffaele@salmaso.org>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

from __future__ import absolute_import, division, print_function, unicode_literals
from stua.version import get_version, HG

VERSION = (1, 0, 1, "final", 0)

__author__ = "Raffaele Salmaso"
__author_email__ = "raffaele@salmaso.org"
__version__ = get_version(VERSION, HG, __file__)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""
    Node and edge feature transformers to extract features for PageXml

    Copyright Xerox(C) 2016 JL. Meunier

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    Developed  for the EU project READ. The READ project has received funding
    from the European Union�s Horizon 2020 research and innovation programme
    under grant agreement No 674943.

"""
import numpy as np

from Transformer import Transformer
from Edge import HorizontalEdge, VerticalEdge, CrossPageEdge

fEPSILON = 10

#------------------------------------------------------------------------------------------------------
class NodeTransformerText(Transformer):
    """
    we will get a list of block and need to send back what a textual feature extractor (TfidfVectorizer) needs.
    So we return a list of strings
    """
    def transform(self, lNode):
        return map(lambda x: x.text, lNode)

#------------------------------------------------------------------------------------------------------
class NodeTransformerTextEnclosed(Transformer):
    """
    we will get a list of block and need to send back what a textual feature extractor (TfidfVectorizer) needs.
    So we return a list of strings
    """
    def transform(self, lNode):
        return map(lambda x: "{%s}"%x.text, lNode) #start/end characters

#------------------------------------------------------------------------------------------------------
class NodeTransformerNeighborText(Transformer):
    """
    for each node, returning the space-separated text of all its neighbor on the page
    """
    def transform(self, lNode):
        return [" ".join([nd2.text for nd2 in nd1.lNeighbor]) for nd1 in lNode]

#------------------------------------------------------------------------------------------------------
class NodeTransformerCrossPageNeighborText(Transformer):
    """
    for each node, returning the space-separated text of all its neighbor on the previous or next page
    """
    def transform(self, lNode):
        return [" ".join([nd2.text for nd2 in nd1.lCPNeighbor]) for nd1 in lNode]

#------------------------------------------------------------------------------------------------------
class NodeTransformerTextLen(Transformer):
    """
    we will get a list of block and need to send back what StandardScaler needs for in-place scaling, a numpy array!.
    So we return a numpy array
    """
    def transform(self, lNode):
        a = np.empty( ( len(lNode), 2 ) , dtype=np.float64)             #--- FEAT #1  text length
        #for i, blk in enumerate(lNode): a[i] = len(blk.text)
        for i, blk in enumerate(lNode):
            a[i,:] = len(blk.text), blk.text.count(' ')
        return a

#------------------------------------------------------------------------------------------------------
class NodeTransformerXYWH(Transformer):
    """
    we will get a list of block and need to send back what StandardScaler needs for in-place scaling, a numpy array!.
    So we return a numpy array
    """
    def transform(self, lNode):
#         a = np.empty( ( len(lNode), 5 ) , dtype=np.float64)
#         for i, blk in enumerate(lNode): a[i, :] = [blk.x1, blk.y2, blk.x2-blk.x1, blk.y2-blk.y1, blk.fontsize]        #--- 2 3 4 5 6
        a = np.empty( ( len(lNode), 2+4+2+4 ) , dtype=np.float64)
        for i, blk in enumerate(lNode):
            page = blk.page
            x1,y1,x2,y2 = blk.x1, blk.y1, blk.x2, blk.y2
            w,h = float(page.w), float(page.h)
            #Normalize by page with and height
            xn1, yn1, xn2, yn2 = x1/w, y1/h, x2/w, y2/h
            #generate X-from-binding
            if page.bEven:
                xb1, xb2    = w - x2    , h - x1
                xnb1, xnb2  = 1.0 - xn2 , 1.0 - xn1
            else:
                xb1, xb2    = x1    , x2
                xnb1, xnb2  = xn1   , xn2
            a[i, :] = [xb1, xb2     , x1, y2, x2-x1, y2-y1   , xnb1, xnb2   , xn1, yn2, xn2-xn1, yn2-yn1]
        return a

#------------------------------------------------------------------------------------------------------
class Node1HotFeatures(Transformer):
    """
    we will get a list of block and return a one-hot encoding, directly
    """
    def transform(self, lNode):
        #We allocate TWO more columns to store in it the tfidf and idf computed at document level.
        #a = np.zeros( ( len(lNode), 10 ) , dtype=np.float64)  # 4 possible orientations: 0, 1, 2, 3
        a = np.zeros( ( len(lNode), 7 ) , dtype=np.float64)  # 4 possible orientations: 0, 1, 2, 3

        for i, blk in enumerate(lNode):
            s = blk.text
            if s.isalnum(): a[i, 0] = 1.0
            if s.isalpha(): a[i, 1] = 1.0
            if s.isdigit(): a[i, 2] = 1.0
            if s.islower(): a[i, 3] = 1.0
            if s.istitle(): a[i, 4] = 1.0
            if s.isupper(): a[i, 5] = 1.0
            if blk.pnum%2 == 0: a[i, 5] = 1.0 #odd/even page number

            #a[i,blk.orientation] = 1.0

        return a

# #------------------------------------------------------------------------------------------------------
# class NodePNumFeatures(Transformer):
#     """
#     does it look like following a plan number pattern after normalization??
#     """
#     def transform(self, lNode):
#         a = np.zeros( ( len(lNode), len(lCRE) ) , dtype=np.float64)
#
#         for i, blk in enumerate(lNode):
#             s, sconf = normalizeNumber(blk.text, blk.sconf)
#             a[i, :] = [1.0 if cre.match(s, 0) else 0.0 for cre in lCRE]
#
#         return a

#------------------------------------------------------------------------------------------------------

# --- EDGES
class EdgeTransformerSourceText(Transformer):
    """
    we will get a list of edges and need to send back what a textual feature extractor (TfidfVectorizer) needs.
    So we return a list of strings, of the source node of the edge
    """
    def transform(self, lEdge):
        #return map(lambda x: x.A.text, lEdge)
        return map(lambda x: "{%s}"%x.A.text, lEdge)

#------------------------------------------------------------------------------------------------------
class EdgeTransformerTargetText(Transformer):
    """
    we will get a list of edges and need to send back what a textual feature extractor (TfidfVectorizer) needs.
    So we return a list of strings, of the source node of the edge
    """
    def transform(self, lEdge):
        #return map(lambda x: x.B.text, lEdge)
        return map(lambda x: "{%s}"%x.B.text, lEdge)

#------------------------------------------------------------------------------------------------------
class Edge1HotFeatures(Transformer):
    """
    we will get a list of edges and return a boolean array, directly

    above/below, left/right, neither but on same page
    same or consecutive pages
    vertical-, horizontal- centered  (at epsilon precision, epsilon typically being 5pt ?)
    left-, top-, right-, bottom- justified  (at epsilon precision)
    TODO sequentiality of content
    TODO crossing ruling-line

    """
    def __init__(self, pageNumSequenciality):
        Transformer.__init__(self)
        self.sqnc = pageNumSequenciality
    def fit(self, x, y=None):
        return self

    def transform(self, lEdge):
        #DISC a = np.zeros( ( len(lEdge), 16 ) , dtype=np.float64)
        a = np.zeros( ( len(lEdge), 27 ) , dtype=np.float64)
        for i, edge in enumerate(lEdge):
            #-- vertical / horizontal / virtual / cross-page / not-neighbor
            if isinstance(edge, VerticalEdge):
                a[i,0] = 1.0
            elif isinstance(edge, HorizontalEdge):
                a[i,1] = 1.0
            elif isinstance(edge, CrossPageEdge):
                a[i,2] = 1.0
            else:
                assert False, "INTERNAL ERROR: unknown type of edge"

            #-- same or consecutive page
            A,B = edge.A, edge.B
            if A.pnum == B.pnum:
                a[i,3] = 1.0

            #-- centering
            if A.x1 + A.x2 - (B.x1 + B.x2) <= 2 * fEPSILON:     #horizontal centered
                a[i, 4] = 1.0
            if A.y1 + A.y2 - (B.y1 + B.y2) <= 2 * fEPSILON:     #V centered
                a[i, 5] = 1.0

            #justified
            if abs(A.x1-B.x1) <= fEPSILON:
                a[i, 6] = 1.0
            if abs(A.y1-B.y1) <= fEPSILON:
                a[i, 7] = 1.0
            if abs(A.x2-B.x2) <= fEPSILON:
                a[i, 8] = 1.0
            if abs(A.y2-B.y2) <= fEPSILON:
                a[i, 9] = 1.0

            #sequenciality, either None, or increasing or decreasing
            sA, sB = A.text, B.text
            if self.sqnc.isPossibleSequence(sA, sB):
                fInSequence = 1.0
            elif self.sqnc.isPossibleSequence(sB, sA):
                fInSequence = -1.0
            else:
                fInSequence = 0.0

            if A.pnum == B.pnum:
                a[i, 10] = fInSequence          #-1, 0, +1
                a[i, 11] = abs(fInSequence)     # 0 or 1
            else:
                a[i, 12] = fInSequence          #-1, 0, +1
                a[i, 13] = abs(fInSequence)     # 0 or 1

            if sA.isalnum(): a[i, 14] = 1.0
            if sA.isalpha(): a[i, 15] = 1.0
            if sA.isdigit(): a[i, 16] = 1.0
            if sA.islower(): a[i, 17] = 1.0
            if sA.istitle(): a[i, 18] = 1.0
            if sA.isupper(): a[i, 19] = 1.0

            if sB.isalnum(): a[i, 20] = 1.0
            if sB.isalpha(): a[i, 21] = 1.0
            if sB.isdigit(): a[i, 22] = 1.0
            if sB.islower(): a[i, 23] = 1.0
            if sB.istitle(): a[i, 24] = 1.0
            if sB.isupper(): a[i, 25] = 1.0

            if sA == sB: a[i, 26] = 1.0

        return a

#------------------------------------------------------------------------------------------------------
class EdgeNumericalSelector(Transformer):
    """
    we will get a list of block and need to send back what StandardScaler needs for in-place scaling, a numpy array!.

    overlap size (ratio of intersection to union of surfaces)
    max(overlap size, 5000)
    identical content in [0, 1] as ratio of lcs to "union"
    max( lcs, 25)
    """
    def transform(self, lEdge):
        #no font size a = np.zeros( ( len(lEdge), 5 ) , dtype=np.float64)
#         a = np.zeros( ( len(lEdge), 7 ) , dtype=np.float64)
        a = np.zeros( ( len(lEdge), 5 ) , dtype=np.float64)
        for i, edge in enumerate(lEdge):
            A,B = edge.A, edge.B

            #overlap
            ovr = A.significantOverlap(B, 0)
            try:
                a[i,0] = ovr / (A.area() + B.area() - ovr)
            except ZeroDivisionError:
                pass
            a[i,1] = min(ovr, 5000.0)

            #
            na, nb = len(A.text), len(B.text)
            lcs = lcs_length(A.text,na, B.text,nb)
            try:
                a[i,2] =  float( lcs / (na+nb-lcs) )
            except ZeroDivisionError:
                pass
            a[i,3] = min(lcs, 50.0)
            a[i,4] = min(lcs, 100.0)

            #if A.txt == B.txt: a[i, 4] = 1.0

#             #fontsize
#             a[i,5] = B.fontsize - A.fontsize
#             a[i,6] = (B.fontsize+1) / (A.fontsize+1)

        return a

# -----------------------------------------------------------------------------------------------------------------------------

def lcs_length(a,na, b,nb):
    """
    Compute the length of the longest common string. Very fast. JLM March 2016

    NOTE: I did not compare against fastlcs...
    """
    #na, nb = len(a), len(b)
    if nb < na: a, na, b, nb = b, nb, a, na
    if na==0: return 0
    na1 = na+1
    curRow  = [0]*na1
    prevRow = [0]*na1
    range1a1 = range(1, na1)
    for i in range(nb):
        bi = b[i]
        prevRow, curRow = curRow, prevRow
        curRow[0] = 0
        curRowj = 0
        for j in range1a1:
            if bi == a[j-1]:
                curRowj = max(1+prevRow[j-1], prevRow[j], curRowj)
            else:
                curRowj = max(prevRow[j], curRowj)
            curRow[j] = curRowj
    return curRowj

<commit_msg>fixed a bug on xb2 feature<commit_after># -*- coding: utf-8 -*-

"""
    Node and edge feature transformers to extract features for PageXml

    Copyright Xerox(C) 2016 JL. Meunier

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.

    Developed  for the EU project READ. The READ project has received funding
    from the European Union�s Horizon 2020 research and innovation programme
    under grant agreement No 674943.

"""
import numpy as np

from Transformer import Transformer
from Edge import HorizontalEdge, VerticalEdge, CrossPageEdge

fEPSILON = 10

#------------------------------------------------------------------------------------------------------
class NodeTransformerText(Transformer):
    """
    we will get a list of block and need to send back what a textual feature extractor (TfidfVectorizer) needs.
    So we return a list of strings
    """
    def transform(self, lNode):
        return map(lambda x: x.text, lNode)

#------------------------------------------------------------------------------------------------------
class NodeTransformerTextEnclosed(Transformer):
    """
    we will get a list of block and need to send back what a textual feature extractor (TfidfVectorizer) needs.
    So we return a list of strings
    """
    def transform(self, lNode):
        return map(lambda x: "{%s}"%x.text, lNode) #start/end characters

#------------------------------------------------------------------------------------------------------
class NodeTransformerNeighborText(Transformer):
    """
    for each node, returning the space-separated text of all its neighbor on the page
    """
    def transform(self, lNode):
        return [" ".join([nd2.text for nd2 in nd1.lNeighbor]) for nd1 in lNode]

#------------------------------------------------------------------------------------------------------
class NodeTransformerCrossPageNeighborText(Transformer):
    """
    for each node, returning the space-separated text of all its neighbor on the previous or next page
    """
    def transform(self, lNode):
        return [" ".join([nd2.text for nd2 in nd1.lCPNeighbor]) for nd1 in lNode]

#------------------------------------------------------------------------------------------------------
class NodeTransformerTextLen(Transformer):
    """
    we will get a list of block and need to send back what StandardScaler needs for in-place scaling, a numpy array!.
    So we return a numpy array
    """
    def transform(self, lNode):
        a = np.empty( ( len(lNode), 2 ) , dtype=np.float64)             #--- FEAT #1  text length
        #for i, blk in enumerate(lNode): a[i] = len(blk.text)
        for i, blk in enumerate(lNode):
            a[i,:] = len(blk.text), blk.text.count(' ')
        return a

#------------------------------------------------------------------------------------------------------
class NodeTransformerXYWH(Transformer):
    """
    we will get a list of block and need to send back what StandardScaler needs for in-place scaling, a numpy array!.
    So we return a numpy array
    """
    def transform(self, lNode):
#         a = np.empty( ( len(lNode), 5 ) , dtype=np.float64)
#         for i, blk in enumerate(lNode): a[i, :] = [blk.x1, blk.y2, blk.x2-blk.x1, blk.y2-blk.y1, blk.fontsize]        #--- 2 3 4 5 6
        a = np.empty( ( len(lNode), 2+4+2+4 ) , dtype=np.float64)
        for i, blk in enumerate(lNode):
            page = blk.page
            x1,y1,x2,y2 = blk.x1, blk.y1, blk.x2, blk.y2
            w,h = float(page.w), float(page.h)
            #Normalize by page with and height
            xn1, yn1, xn2, yn2 = x1/w, y1/h, x2/w, y2/h
            #generate X-from-binding
            if page.bEven:
                xb1, xb2    = w - x2    , w - x1
                xnb1, xnb2  = 1.0 - xn2 , 1.0 - xn1
            else:
                xb1, xb2    = x1    , x2
                xnb1, xnb2  = xn1   , xn2
            a[i, :] = [xb1, xb2     , x1, y2, x2-x1, y2-y1   , xnb1, xnb2   , xn1, yn2, xn2-xn1, yn2-yn1]
        return a

#------------------------------------------------------------------------------------------------------
class Node1HotFeatures(Transformer):
    """
    we will get a list of block and return a one-hot encoding, directly
    """
    def transform(self, lNode):
        #We allocate TWO more columns to store in it the tfidf and idf computed at document level.
        #a = np.zeros( ( len(lNode), 10 ) , dtype=np.float64)  # 4 possible orientations: 0, 1, 2, 3
        a = np.zeros( ( len(lNode), 7 ) , dtype=np.float64)  # 4 possible orientations: 0, 1, 2, 3

        for i, blk in enumerate(lNode):
            s = blk.text
            if s.isalnum(): a[i, 0] = 1.0
            if s.isalpha(): a[i, 1] = 1.0
            if s.isdigit(): a[i, 2] = 1.0
            if s.islower(): a[i, 3] = 1.0
            if s.istitle(): a[i, 4] = 1.0
            if s.isupper(): a[i, 5] = 1.0
            if blk.pnum%2 == 0: a[i, 5] = 1.0 #odd/even page number

            #a[i,blk.orientation] = 1.0

        return a

# #------------------------------------------------------------------------------------------------------
# class NodePNumFeatures(Transformer):
#     """
#     does it look like following a plan number pattern after normalization??
#     """
#     def transform(self, lNode):
#         a = np.zeros( ( len(lNode), len(lCRE) ) , dtype=np.float64)
#
#         for i, blk in enumerate(lNode):
#             s, sconf = normalizeNumber(blk.text, blk.sconf)
#             a[i, :] = [1.0 if cre.match(s, 0) else 0.0 for cre in lCRE]
#
#         return a

#------------------------------------------------------------------------------------------------------

# --- EDGES
class EdgeTransformerSourceText(Transformer):
    """
    we will get a list of edges and need to send back what a textual feature extractor (TfidfVectorizer) needs.
    So we return a list of strings, of the source node of the edge
    """
    def transform(self, lEdge):
        #return map(lambda x: x.A.text, lEdge)
        return map(lambda x: "{%s}"%x.A.text, lEdge)

#------------------------------------------------------------------------------------------------------
class EdgeTransformerTargetText(Transformer):
    """
    we will get a list of edges and need to send back what a textual feature extractor (TfidfVectorizer) needs.
    So we return a list of strings, of the source node of the edge
    """
    def transform(self, lEdge):
        #return map(lambda x: x.B.text, lEdge)
        return map(lambda x: "{%s}"%x.B.text, lEdge)

#------------------------------------------------------------------------------------------------------
class Edge1HotFeatures(Transformer):
    """
    we will get a list of edges and return a boolean array, directly

    above/below, left/right, neither but on same page
    same or consecutive pages
    vertical-, horizontal- centered  (at epsilon precision, epsilon typically being 5pt ?)
    left-, top-, right-, bottom- justified  (at epsilon precision)
    TODO sequentiality of content
    TODO crossing ruling-line

    """
    def __init__(self, pageNumSequenciality):
        Transformer.__init__(self)
        self.sqnc = pageNumSequenciality
    def fit(self, x, y=None):
        return self

    def transform(self, lEdge):
        #DISC a = np.zeros( ( len(lEdge), 16 ) , dtype=np.float64)
        a = np.zeros( ( len(lEdge), 27 ) , dtype=np.float64)
        for i, edge in enumerate(lEdge):
            #-- vertical / horizontal / virtual / cross-page / not-neighbor
            if isinstance(edge, VerticalEdge):
                a[i,0] = 1.0
            elif isinstance(edge, HorizontalEdge):
                a[i,1] = 1.0
            elif isinstance(edge, CrossPageEdge):
                a[i,2] = 1.0
            else:
                assert False, "INTERNAL ERROR: unknown type of edge"

            #-- same or consecutive page
            A,B = edge.A, edge.B
            if A.pnum == B.pnum:
                a[i,3] = 1.0

            #-- centering
            if A.x1 + A.x2 - (B.x1 + B.x2) <= 2 * fEPSILON:     #horizontal centered
                a[i, 4] = 1.0
            if A.y1 + A.y2 - (B.y1 + B.y2) <= 2 * fEPSILON:     #V centered
                a[i, 5] = 1.0

            #justified
            if abs(A.x1-B.x1) <= fEPSILON:
                a[i, 6] = 1.0
            if abs(A.y1-B.y1) <= fEPSILON:
                a[i, 7] = 1.0
            if abs(A.x2-B.x2) <= fEPSILON:
                a[i, 8] = 1.0
            if abs(A.y2-B.y2) <= fEPSILON:
                a[i, 9] = 1.0

            #sequenciality, either None, or increasing or decreasing
            sA, sB = A.text, B.text
            if self.sqnc.isPossibleSequence(sA, sB):
                fInSequence = 1.0
            elif self.sqnc.isPossibleSequence(sB, sA):
                fInSequence = -1.0
            else:
                fInSequence = 0.0

            if A.pnum == B.pnum:
                a[i, 10] = fInSequence          #-1, 0, +1
                a[i, 11] = abs(fInSequence)     # 0 or 1
            else:
                a[i, 12] = fInSequence          #-1, 0, +1
                a[i, 13] = abs(fInSequence)     # 0 or 1

            if sA.isalnum(): a[i, 14] = 1.0
            if sA.isalpha(): a[i, 15] = 1.0
            if sA.isdigit(): a[i, 16] = 1.0
            if sA.islower(): a[i, 17] = 1.0
            if sA.istitle(): a[i, 18] = 1.0
            if sA.isupper(): a[i, 19] = 1.0

            if sB.isalnum(): a[i, 20] = 1.0
            if sB.isalpha(): a[i, 21] = 1.0
            if sB.isdigit(): a[i, 22] = 1.0
            if sB.islower(): a[i, 23] = 1.0
            if sB.istitle(): a[i, 24] = 1.0
            if sB.isupper(): a[i, 25] = 1.0

            if sA == sB: a[i, 26] = 1.0

        return a

#------------------------------------------------------------------------------------------------------
class EdgeNumericalSelector(Transformer):
    """
    we will get a list of block and need to send back what StandardScaler needs for in-place scaling, a numpy array!.

    overlap size (ratio of intersection to union of surfaces)
    max(overlap size, 5000)
    identical content in [0, 1] as ratio of lcs to "union"
    max( lcs, 25)
    """
    def transform(self, lEdge):
        #no font size a = np.zeros( ( len(lEdge), 5 ) , dtype=np.float64)
#         a = np.zeros( ( len(lEdge), 7 ) , dtype=np.float64)
        a = np.zeros( ( len(lEdge), 5 ) , dtype=np.float64)
        for i, edge in enumerate(lEdge):
            A,B = edge.A, edge.B

            #overlap
            ovr = A.significantOverlap(B, 0)
            try:
                a[i,0] = ovr / (A.area() + B.area() - ovr)
            except ZeroDivisionError:
                pass
            a[i,1] = min(ovr, 5000.0)

            #
            na, nb = len(A.text), len(B.text)
            lcs = lcs_length(A.text,na, B.text,nb)
            try:
                a[i,2] =  float( lcs / (na+nb-lcs) )
            except ZeroDivisionError:
                pass
            a[i,3] = min(lcs, 50.0)
            a[i,4] = min(lcs, 100.0)

            #if A.txt == B.txt: a[i, 4] = 1.0

#             #fontsize
#             a[i,5] = B.fontsize - A.fontsize
#             a[i,6] = (B.fontsize+1) / (A.fontsize+1)

        return a

# -----------------------------------------------------------------------------------------------------------------------------

def lcs_length(a,na, b,nb):
    """
    Compute the length of the longest common string. Very fast. JLM March 2016

    NOTE: I did not compare against fastlcs...
    """
    #na, nb = len(a), len(b)
    if nb < na: a, na, b, nb = b, nb, a, na
    if na==0: return 0
    na1 = na+1
    curRow  = [0]*na1
    prevRow = [0]*na1
    range1a1 = range(1, na1)
    for i in range(nb):
        bi = b[i]
        prevRow, curRow = curRow, prevRow
        curRow[0] = 0
        curRowj = 0
        for j in range1a1:
            if bi == a[j-1]:
                curRowj = max(1+prevRow[j-1], prevRow[j], curRowj)
            else:
                curRowj = max(prevRow[j], curRowj)
            curRow[j] = curRowj
    return curRowj

<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import pysam

if len(sys.argv) == 4:
    tbx = pysam.Tabixfile(sys.argv[2])

    with open(sys.argv[1]) as l1seq:
        for line in l1seq:

            if line.startswith('UUID'):
                header = line.strip().split()
                header.append(sys.argv[3])
                print '\t'.join(header)
                continue

            c = line.strip().split()
            chrom = c[1]
            start = int(c[2])
            end   = int(c[3])

            annotations = []

            if chrom in tbx.contigs:
                for rec in tbx.fetch(chrom, start, end):
                    annotations.append('|'.join(rec.strip().split()))

            annotations = list(set(annotations)) # uniqify

            if len(annotations) == 0: annotations.append('NA')

            print line.strip() + '\t' + ','.join(annotations)

else:
    sys.exit("usage: %s <l1seq.py output> <tabix> <header name>" % sys.argv[0])
<commit_msg>improve annotator script<commit_after>#!/usr/bin/env python

import sys
import pysam
import argparse

def main(args):
    tbx = pysam.Tabixfile(args.tabix)

    with open(args.table) as l1seq:
        for line in l1seq:

            if line.startswith('UUID'):
                header = line.strip().split()
                header.append(args.name)
                print '\t'.join(header)
                continue

            c = line.strip().split()
            chrom = c[1]
            start = int(c[2])
            end   = int(c[3])

            annotations = []

            if chrom in tbx.contigs:
                for rec in tbx.fetch(chrom, start, end):
                    if args.nonref:
                        sfam = c[6]
                        annot = rec.strip().split()
                        if annot[3] == sfam:
                            annotations.append('|'.join(annot))
                    else:
                        annotations.append('|'.join(rec.strip().split()))

            annotations = list(set(annotations)) # uniqify

            if len(annotations) == 0: annotations.append('NA')

            print line.strip() + '\t' + ','.join(annotations)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='annotate tebreak table')
    parser.add_argument('-t', '--table', required=True)
    parser.add_argument('-x', '--tabix', required=True)
    parser.add_argument('-n', '--name', required=True)
    parser.add_argument('--nonref', default=False, action='store_true', help='annotation mode specific to non-reference insertions')
    args = parser.parse_args()
    main(args)
<|endoftext|>


<commit_before>#! /usr/bin/env python3
#
# Copyright 2017 Linaro Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Assemble multiple images into a single image that can be flashed on the device.
"""

import argparse
import errno
import io
import re
import os.path

def same_keys(a, b):
    """Determine if the dicts a and b have the same keys in them"""
    for ak in a.keys():
        if ak not in b:
            return False
    for bk in b.keys():
        if bk not in a:
            return False
    return True

offset_re = re.compile(r"^#define DT_FLASH_AREA_([0-9A-Z_]+)_OFFSET(_0)?\s+(0x[0-9a-fA-F]+|[0-9]+)$")
size_re   = re.compile(r"^#define DT_FLASH_AREA_([0-9A-Z_]+)_SIZE(_0)?\s+(0x[0-9a-fA-F]+|[0-9]+)$")

class Assembly():
    def __init__(self, output, bootdir):
        self.find_slots(bootdir)
        try:
            os.unlink(output)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
        self.output = output

    def find_slots(self, bootdir):
        offsets = {}
        sizes = {}
        with open(os.path.join(bootdir, 'zephyr', 'include', 'generated', 'devicetree_legacy_unfixed.h'), 'r') as fd:
            for line in fd:
                m = offset_re.match(line)
                if m is not None:
                    offsets[m.group(1)] = int(m.group(3), 0)
                m = size_re.match(line)
                if m is not None:
                    sizes[m.group(1)] = int(m.group(3), 0)

        if not same_keys(offsets, sizes):
            raise Exception("Inconsistent data in devicetree.h")

        # We care about the MCUBOOT, IMAGE_0, and IMAGE_1 partitions.
        if 'MCUBOOT' not in offsets:
            raise Exception("Board partition table does not have mcuboot partition")

        if 'IMAGE_0' not in offsets:
            raise Exception("Board partition table does not have image-0 partition")

        if 'IMAGE_1' not in offsets:
            raise Exception("Board partition table does not have image-1 partition")

        self.offsets = offsets
        self.sizes = sizes

    def add_image(self, source, partition):
        with open(self.output, 'ab') as ofd:
            pos = ofd.tell()
            print("partition {}, pos={}, offset={}".format(partition, pos, self.offsets[partition]))
            if pos > self.offsets[partition]:
                raise Exception("Partitions not in order, unsupported")
            if pos < self.offsets[partition]:
                buf = b'\xFF' * (self.offsets[partition] - pos)
                ofd.write(buf)
            with open(source, 'rb') as rfd:
                ibuf = rfd.read()
                if len(ibuf) > self.sizes[partition]:
                    raise Exception("Image {} is too large for partition".format(source))
            ofd.write(ibuf)

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('-b', '--bootdir', required=True,
            help='Directory of built bootloader')
    parser.add_argument('-p', '--primary', required=True,
            help='Signed image file for primary image')
    parser.add_argument('-s', '--secondary',
            help='Signed image file for secondary image')
    parser.add_argument('-o', '--output', required=True,
            help='Filename to write full image to')

    args = parser.parse_args()
    output = Assembly(args.output, args.bootdir)

    output.add_image(os.path.join(args.bootdir, 'zephyr', 'zephyr.bin'), 'MCUBOOT')
    output.add_image(args.primary, "IMAGE_0")
    if args.secondary is not None:
        output.add_image(args.secondary, "IMAGE_1")

if __name__ == '__main__':
    main()
<commit_msg>scripts/assemble: Rework to use EDT library to get devicetree data<commit_after>#! /usr/bin/env python3
#
# Copyright 2017 Linaro Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Assemble multiple images into a single image that can be flashed on the device.
"""

import argparse
import errno
import io
import re
import os.path
import sys

ZEPHYR_BASE = os.getenv("ZEPHYR_BASE")
if not ZEPHYR_BASE:
    sys.exit("$ZEPHYR_BASE environment variable undefined")

sys.path.insert(0, os.path.join(ZEPHYR_BASE, "scripts", "dts"))
import edtlib

def same_keys(a, b):
    """Determine if the dicts a and b have the same keys in them"""
    for ak in a.keys():
        if ak not in b:
            return False
    for bk in b.keys():
        if bk not in a:
            return False
    return True

offset_re = re.compile(r"^#define DT_FLASH_AREA_([0-9A-Z_]+)_OFFSET(_0)?\s+(0x[0-9a-fA-F]+|[0-9]+)$")
size_re   = re.compile(r"^#define DT_FLASH_AREA_([0-9A-Z_]+)_SIZE(_0)?\s+(0x[0-9a-fA-F]+|[0-9]+)$")

class Assembly():
    def __init__(self, output, bootdir, edt):
        self.find_slots(edt)
        try:
            os.unlink(output)
        except OSError as e:
            if e.errno != errno.ENOENT:
                raise
        self.output = output

    def find_slots(self, edt):
        offsets = {}
        sizes = {}

        part_nodes = edt.compat2nodes["fixed-partitions"]
        for node in part_nodes:
            for child in node.children.values():
                if "label" in child.props:
                    label = child.props["label"].val
                    offsets[label] = child.regs[0].addr
                    sizes[label] = child.regs[0].size

        if not same_keys(offsets, sizes):
            raise Exception("Inconsistent data in devicetree.h")

        # We care about the mcuboot, image-0, and image-1 partitions.
        if 'mcuboot' not in offsets:
            raise Exception("Board partition table does not have mcuboot partition")

        if 'image-0' not in offsets:
            raise Exception("Board partition table does not have image-0 partition")

        if 'image-1' not in offsets:
            raise Exception("Board partition table does not have image-1 partition")

        self.offsets = offsets
        self.sizes = sizes

    def add_image(self, source, partition):
        with open(self.output, 'ab') as ofd:
            pos = ofd.tell()
            print("partition {}, pos={}, offset={}".format(partition, pos, self.offsets[partition]))
            if pos > self.offsets[partition]:
                raise Exception("Partitions not in order, unsupported")
            if pos < self.offsets[partition]:
                buf = b'\xFF' * (self.offsets[partition] - pos)
                ofd.write(buf)
            with open(source, 'rb') as rfd:
                ibuf = rfd.read()
                if len(ibuf) > self.sizes[partition]:
                    raise Exception("Image {} is too large for partition".format(source))
            ofd.write(ibuf)

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument('-b', '--bootdir', required=True,
            help='Directory of built bootloader')
    parser.add_argument('-p', '--primary', required=True,
            help='Signed image file for primary image')
    parser.add_argument('-s', '--secondary',
            help='Signed image file for secondary image')
    parser.add_argument('-o', '--output', required=True,
            help='Filename to write full image to')

    args = parser.parse_args()

    # Extract board name from path
    board = os.path.split(os.path.split(args.bootdir)[0])[1]

    dts_path = os.path.join(args.bootdir, "zephyr", board + ".dts.pre.tmp")

    edt = edtlib.EDT(dts_path, [os.path.join(ZEPHYR_BASE, "dts", "bindings")],
            warn_reg_unit_address_mismatch=False)

    output = Assembly(args.output, args.bootdir, edt)

    output.add_image(os.path.join(args.bootdir, 'zephyr', 'zephyr.bin'), 'mcuboot')
    output.add_image(args.primary, "image-0")
    if args.secondary is not None:
        output.add_image(args.secondary, "image-1")

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/python
#
# Build the entire ISAAC Project
#
#
#

import subprocess
import os
import sys

projects = ['va-ochre',
			'va-isaac-metadata',
			'va-isaac-mojo',
			'va-newtons-cradle',
			'va-logic',
			'va-query-service',
			'va-isaac-gui',
			'va-solor-goods',
			'va-expression-service',
			'va-isaac-gui-pa']

args = ['-e', 'clean']

def mvn(*args):
	print("Running " )
	print(args)
	print(" in " + os.getcwd())
	mvnLocation = "C:\\Program Files\\Maven\\bin\\mvn.bat"
	return print("Output: " + subprocess.check_call([mvnLocation] + list(args)))

for project in projects:
        if os.path.isdir(os.getcwd() + project)== False:
                cwd = os.getcwd()
                print("In: " + cwd + " Entering project " + project)
                print("Enviorment: " + os.environ['JAVA_HOME'])
                os.chdir(project)

                if project == 'va-expression-service' or project == 'va-isaac-gui-pa':
                        args.extend(['package'])
                else:
                        args.extend(['install'])

                if(len(sys.argv) > 1):
                        if(str(sys.argv[1]) == 'skipTest'):
                                args.extend(['-DskipTests'])
                print ("Build Argument")
                print (args)

                #This fails the build, if it results in a non-0 exit status
                mvn(args)

                os.chdir(os.pardir)
        else:
                 print(project + " folder does not exists in " + os.getcwd())
<commit_msg>Revert "Fixed a windows bug that prevented run locating the Maven batch file to run mvn commands. This will need to be modified to run on Linux"<commit_after>#! /usr/bin/python
#
# Build the entire ISAAC Project
#
#
#

import subprocess
import os
import sys

projects = ['va-isaac-parent',
			'va-ochre',
			'va-isaac-metadata',
			'va-isaac-mojo',
			'va-newtons-cradle',
			'va-logic',
			'va-query-service',
			'va-isaac-gui',
			'va-solor-goods',
			'va-expression-service',
			'va-isaac-gui-pa']

defaultArgs = ['-e', 'clean']

def mvn(args):
	return subprocess.check_call(['mvn'] + args)

for project in projects:
	cwd = os.getcwd()
	print("In: " + cwd + " Entering project " + project)
	os.chdir(project)

	args = defaultArgs[:]
	if project == 'va-expression-service' or project == 'va-isaac-gui-pa':
		args.extend(['package'])
	else:
		args.extend(['install'])

	print ("Build Argument")
	print (args)

	#This fails the build, if it results in a non-0 exit status
	mvn(args)

	os.chdir(os.pardir)<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""This script fetches and prepares an SDK chroot.
"""

import optparse
import os
import sys
import urlparse

from chromite.buildbot import constants
from chromite.lib import cgroups
from chromite.lib import cros_build_lib
from chromite.lib import locking
from chromite.lib import sudo

cros_build_lib.STRICT_SUDO = True

DEFAULT_URL = 'https://commondatastorage.googleapis.com/chromiumos-sdk/'
SRC_ROOT = os.path.realpath(constants.SOURCE_ROOT)
SDK_DIR = os.path.join(SRC_ROOT, 'sdks')
OVERLAY_DIR = os.path.join(SRC_ROOT, 'src/third_party/chromiumos-overlay')
SDK_VERSION_FILE = os.path.join(OVERLAY_DIR,
                                'chromeos/binhost/host/sdk_version.conf')

# TODO(zbehan): Remove the dependency on these, reimplement them in python
MAKE_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/make_chroot.sh')]
ENTER_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/enter_chroot.sh')]

# We need these tools to run. Very common tools (tar,..) are ommited.
NEEDED_TOOLS = ['curl']

def GetHostArch():
  """Returns a string for the host architecture"""
  out = cros_build_lib.RunCommand(['uname', '-m'],
      redirect_stdout=True, print_cmd=False).output
  return out.rstrip('\n')

def CheckPrerequisites(needed_tools):
  """Verifies that the required tools are present on the system.

  This is especially important as this script is intended to run
  outside the chroot.

  Arguments:
    needed_tools: an array of string specified binaries to look for.

  Returns:
    True if all needed tools were found.
  """
  missing = []
  for tool in needed_tools:
    cmd = ['which', tool]
    try:
      cros_build_lib.RunCommand(cmd, print_cmd=False, redirect_stdout=True,
                                combine_stdout_stderr=True)
    except cros_build_lib.RunCommandError:
      missing.append(tool)
  return missing

def GetLatestVersion():
  """Extracts latest version from chromiumos-overlay."""
  sdk_file = open(SDK_VERSION_FILE)
  buf = sdk_file.readline().rstrip('\n').split('=')
  if buf[0] != 'SDK_LATEST_VERSION':
    raise Exception('Malformed version file')
  return buf[1].strip('"')

def GetArchStageTarball(tarballArch, version):
  """Returns the URL for a given arch/version"""
  D = { 'x86_64': 'cros-sdk-' }
  try:
    return DEFAULT_URL + D[tarballArch] + version + '.tbz2'
  except KeyError:
    raise SystemExit('Unsupported arch: %s' % (tarballArch,))

def FetchRemoteTarball(url):
  """Fetches a tarball given by url, and place it in sdk/."""

  def RunCurl(args, **kwargs):
    """Runs curl and wraps around all necessary hacks."""
    cmd = ['curl']
    cmd.extend(args)

    result = cros_build_lib.RunCommand(cmd, error_ok=True, **kwargs)
    if result.returncode > 0:
      # These are the return codes of failing certs as per 'man curl'.
      if result.returncode in (51, 58, 60):
        print 'Download failed with certificate error? Try "sudo c_rehash".'
      else:
        print 'Curl failed!'
      sys.exit(1)

    return result

  tarball_name = os.path.basename(urlparse.urlparse(url).path)
  tarball_dest = os.path.join(SDK_DIR, tarball_name)

  # Cleanup old tarballs.
  files_to_delete = [f for f in os.listdir(SDK_DIR) if f != tarball_name]
  if files_to_delete:
    print 'Cleaning up old tarballs: ' + str(files_to_delete)
    for f in files_to_delete:
      f_path = os.path.join(SDK_DIR, f)
      # Only delete regular files that belong to us.
      if os.path.isfile(f_path) and os.stat(f_path).st_uid == os.getuid():
        os.remove(f_path)

  curl_opts = ['-f', '--retry', '5', '-L', '-y', '30',
               '--output', tarball_dest]
  print 'Downloading sdk: "%s"' % url
  if not url.startswith('file://') and os.path.exists(tarball_dest):
    # Only resume for remote URLs. If the file is local, there's no
    # real speedup, and using the same filename for different files
    # locally will cause issues.
    curl_opts.extend(['-C', '-'])

    # Additionally, certain versions of curl incorrectly fail if
    # told to resume a file that is fully downloaded, thus do a
    # check on our own.
    # see:
    # https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3482927&group_id=976
    result = RunCurl(['-I', url],
                     redirect_stdout=True,
                     redirect_stderr=True,
                     print_cmd=False)

    for x in result.output.splitlines():
      if x.lower().startswith("content-length:"):
        length = int(x.split(":", 1)[-1].strip())
        if length == os.path.getsize(tarball_dest):
          # Fully fetched; bypass invoking curl, since it can screw up handling
          # of this (>=7.21.4 and up).
          return tarball_dest
        break
  curl_opts.append(url)
  RunCurl(curl_opts)
  return tarball_dest

def BootstrapChroot(chroot_path, stage_url, replace):
  """Builds a new chroot from source"""
  cmd = MAKE_CHROOT + ['--chroot', chroot_path,
                       '--nousepkg']

  stage = None
  if stage_url:
    stage = FetchRemoteTarball(stage_url)

  if stage:
    cmd.extend(['--stage3_path', stage])

  if replace:
    cmd.append('--replace')

  try:
    cros_build_lib.RunCommand(cmd, print_cmd=False)
  except cros_build_lib.RunCommandError:
    raise SystemExit('Running %r failed!' % cmd)

def CreateChroot(sdk_url, sdk_version, chroot_path, replace):
  """Creates a new chroot from a given SDK"""
  if not os.path.exists(SDK_DIR):
    cros_build_lib.RunCommand(['mkdir', '-p', SDK_DIR], print_cmd=False)

  # Based on selections, fetch the tarball
  if sdk_url:
    url = sdk_url
  else:
    arch = GetHostArch()
    if sdk_version:
      url = GetArchStageTarball(arch, sdk_version)
    else:
      url = GetArchStageTarball(arch)

  sdk = FetchRemoteTarball(url)

  # TODO(zbehan): Unpack and install
  # For now, we simply call make_chroot on the prebuilt chromeos-sdk.
  # make_chroot provides a variety of hacks to make the chroot useable.
  # These should all be eliminated/minimised, after which, we can change
  # this to just unpacking the sdk.
  cmd = MAKE_CHROOT + ['--stage3_path', sdk,
                       '--chroot', chroot_path]

  if replace:
    cmd.append('--replace')

  try:
    cros_build_lib.RunCommand(cmd, print_cmd=False)
  except cros_build_lib.RunCommandError:
    raise SystemExit('Running %r failed!' % cmd)

def DeleteChroot(chroot_path):
  """Deletes an existing chroot"""
  cmd = MAKE_CHROOT + ['--chroot', chroot_path,
                       '--delete']
  try:
    cros_build_lib.RunCommand(cmd, print_cmd=False)
  except cros_build_lib.RunCommandError:
    raise SystemExit('Running %r failed!' % cmd)

def _CreateLockFile(path):
   """Create a lockfile via sudo that is writable by current user."""
   cros_build_lib.SudoRunCommand(['touch', path], print_cmd=False)
   cros_build_lib.SudoRunCommand(['chown', str(os.getuid()), path],
                                 print_cmd=False)
   cros_build_lib.SudoRunCommand(['chmod', '644', path], print_cmd=False)

def EnterChroot(chroot_path, chrome_root, chrome_root_mount, additional_args):
  """Enters an existing SDK chroot"""
  cmd = ENTER_CHROOT + ['--chroot', chroot_path]
  if chrome_root:
    cmd.extend(['--chrome_root', chrome_root])
  if chrome_root_mount:
    cmd.extend(['--chrome_root_mount', chrome_root_mount])
  if len(additional_args) > 0:
    cmd.append('--')
    cmd.extend(additional_args)

  ret = cros_build_lib.RunCommand(cmd, print_cmd=False, error_code_ok=True)
  # If we were in interactive mode, ignore the exit code; it'll be whatever
  # they last ran w/in the chroot and won't matter to us one way or another.
  # Note this does allow chroot entrance to fail and be ignored during
  # interactive; this is however a rare case and the user will immediately
  # see it (nor will they be checking the exit code manually).
  if ret.returncode != 0 and additional_args:
    raise SystemExit('Running %r failed with exit code %i'
                     % (cmd, ret.returncode))

def main(argv):
  # TODO(ferringb): make argv required once depot_tools is fixed.
  usage="""usage: %prog [options] [VAR1=val1 .. VARn=valn -- <args>]

This script manages a local CrOS SDK chroot. Depending on the flags,
it can download, build or enter a chroot.

Action taken is the following:
--enter  (default)  .. Installs and enters a chroot
--download          .. Just download a chroot (enter if combined with --enter)
--bootstrap         .. Builds a chroot from source (enter if --enter)
--delete            .. Removes a chroot
"""
  sdk_latest_version = GetLatestVersion()
  parser = optparse.OptionParser(usage)
  # Actions:
  parser.add_option('', '--bootstrap',
                    action='store_true', dest='bootstrap', default=False,
                    help=('Build a new SDK chroot from source'))
  parser.add_option('', '--delete',
                    action='store_true', dest='delete', default=False,
                    help=('Delete the current SDK chroot'))
  parser.add_option('', '--download',
                    action='store_true', dest='download', default=False,
                    help=('Download and install a prebuilt SDK'))
  parser.add_option('', '--enter',
                    action='store_true', dest='enter', default=False,
                    help=('Enter the SDK chroot, possibly (re)create first'))

  # Global options:
  parser.add_option('', '--chroot',
                    dest='chroot', default=constants.DEFAULT_CHROOT_DIR,
                    help=('SDK chroot dir name [%s]' %
                          constants.DEFAULT_CHROOT_DIR))

  # Additional options:
  parser.add_option('', '--chrome_root',
                    dest='chrome_root', default='',
                    help=('Mount this chrome root into the SDK chroot'))
  parser.add_option('', '--chrome_root_mount',
                    dest='chrome_root_mount', default='',
                    help=('Mount chrome into this path inside SDK chroot'))
  parser.add_option('-r', '--replace',
                    action='store_true', dest='replace', default=False,
                    help=('Replace an existing SDK chroot'))
  parser.add_option('-u', '--url',
                    dest='sdk_url', default='',
                    help=('''Use sdk tarball located at this url.
                             Use file:// for local files.'''))
  parser.add_option('-v', '--version',
                    dest='sdk_version', default='',
                    help=('Use this sdk version [%s]' % sdk_latest_version))
  (options, remaining_arguments) = parser.parse_args(argv)

  # Some sanity checks first, before we ask for sudo credentials.
  if cros_build_lib.IsInsideChroot():
    parser.error("This needs to be ran outside the chroot")

  missing = CheckPrerequisites(NEEDED_TOOLS)
  if missing:
    parser.error((
        'The tool(s) %s were not found.'
        'Please install the appropriate package in your host.'
        'Example(ubuntu):'
        '  sudo apt-get install <packagename>'
        % (', '.join(missing))))

  # Default action is --enter, if no other is selected.
  if not (options.bootstrap or options.download or options.delete):
    options.enter = True

  # Only --enter can process additional args as passthrough commands.
  # Warn and exit for least surprise.
  if len(remaining_arguments) > 0 and not options.enter:
    parser.error("Additional arguments are not permitted, unless running "
                 "with --enter")

  # Some actions can be combined, as they merely modify how is the chroot
  # going to be made. The only option that hates all others is --delete.
  if options.delete and \
    (options.enter or options.download or options.bootstrap):
    parser.error("--delete cannot be combined with --enter, "
                 "--download or --bootstrap")
  # NOTE: --delete is a true hater, it doesn't like other options either, but
  # those will hardly lead to confusion. Nobody can expect to pass --version to
  # delete and actually change something.

  if options.bootstrap and options.download:
    parser.error("Either --bootstrap or --download, not both")

  # Bootstrap will start off from a non-selectable stage3 tarball. Attempts to
  # select sdk by version are confusing. Warn and exit. We can still specify a
  # tarball by path or URL though.
  if options.bootstrap and options.sdk_version:
    parser.error("Cannot use --version when bootstrapping")

  chroot_path = os.path.join(SRC_ROOT, options.chroot)
  chroot_path = os.path.abspath(chroot_path)
  chroot_path = os.path.normpath(chroot_path)

  if not options.sdk_version:
    sdk_version = sdk_latest_version
  else:
    sdk_version = options.sdk_version

  if options.delete and not os.path.exists(chroot_path):
    print "Not doing anything. The chroot you want to remove doesn't exist."
    return 0

  lock_path = os.path.dirname(chroot_path)
  lock_path = os.path.join(lock_path,
                           '.%s_lock' % os.path.basename(chroot_path))
  with sudo.SudoKeepAlive():
    with cgroups.SimpleContainChildren('cros_sdk'):
      _CreateLockFile(lock_path)
      with locking.FileLock(lock_path, 'chroot lock') as lock:
        if options.delete:
          lock.write_lock()
          DeleteChroot(chroot_path)
          return 0

        # Print a suggestion for replacement, but not if running just --enter.
        if os.path.exists(chroot_path) and not options.replace and \
            (options.bootstrap or options.download):
          print "Chroot already exists. Run with --replace to re-create."

        # Chroot doesn't exist or asked to replace.
        if not os.path.exists(chroot_path) or options.replace:
          lock.write_lock()
          if options.bootstrap:
            BootstrapChroot(chroot_path, options.sdk_url,
                            options.replace)
          else:
            CreateChroot(options.sdk_url, sdk_version,
                         chroot_path, options.replace)
        if options.enter:
          lock.read_lock()
          EnterChroot(chroot_path, options.chrome_root,
                      options.chrome_root_mount, remaining_arguments)
<commit_msg>cros_sdk: support downloading both xz and bz2 images<commit_after>#!/usr/bin/env python
# Copyright (c) 2011-2012 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""This script fetches and prepares an SDK chroot.
"""

import optparse
import os
import sys
import urlparse

from chromite.buildbot import constants
from chromite.lib import cgroups
from chromite.lib import cros_build_lib
from chromite.lib import locking
from chromite.lib import sudo

cros_build_lib.STRICT_SUDO = True

DEFAULT_URL = 'https://commondatastorage.googleapis.com/chromiumos-sdk/'
SDK_SUFFIXES = ['.tbz2', '.tar.xz']

SRC_ROOT = os.path.realpath(constants.SOURCE_ROOT)
SDK_DIR = os.path.join(SRC_ROOT, 'sdks')
OVERLAY_DIR = os.path.join(SRC_ROOT, 'src/third_party/chromiumos-overlay')
SDK_VERSION_FILE = os.path.join(OVERLAY_DIR,
                                'chromeos/binhost/host/sdk_version.conf')

# TODO(zbehan): Remove the dependency on these, reimplement them in python
MAKE_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/make_chroot.sh')]
ENTER_CHROOT = [os.path.join(SRC_ROOT, 'src/scripts/sdk_lib/enter_chroot.sh')]

# We need these tools to run. Very common tools (tar,..) are ommited.
NEEDED_TOOLS = ['curl']

def GetHostArch():
  """Returns a string for the host architecture"""
  out = cros_build_lib.RunCommand(['uname', '-m'],
      redirect_stdout=True, print_cmd=False).output
  return out.rstrip('\n')

def CheckPrerequisites(needed_tools):
  """Verifies that the required tools are present on the system.

  This is especially important as this script is intended to run
  outside the chroot.

  Arguments:
    needed_tools: an array of string specified binaries to look for.

  Returns:
    True if all needed tools were found.
  """
  missing = []
  for tool in needed_tools:
    cmd = ['which', tool]
    try:
      cros_build_lib.RunCommand(cmd, print_cmd=False, redirect_stdout=True,
                                combine_stdout_stderr=True)
    except cros_build_lib.RunCommandError:
      missing.append(tool)
  return missing

def GetLatestVersion():
  """Extracts latest version from chromiumos-overlay."""
  sdk_file = open(SDK_VERSION_FILE)
  buf = sdk_file.readline().rstrip('\n').split('=')
  if buf[0] != 'SDK_LATEST_VERSION':
    raise Exception('Malformed version file')
  return buf[1].strip('"')

def GetArchStageTarballs(tarballArch, version):
  """Returns the URL for a given arch/version"""
  D = { 'x86_64': 'cros-sdk-' }
  try:
    return [DEFAULT_URL + D[tarballArch] + version + x for x in SDK_SUFFIXES]
  except KeyError:
    raise SystemExit('Unsupported arch: %s' % (tarballArch,))

def FetchRemoteTarballs(urls):
  """Fetches a tarball given by url, and place it in sdk/.

  Args:
    urls: List of URLs to try to download. Download will stop on first success.

  Returns:
    Full path to the downloaded file
  """

  def RunCurl(args, **kwargs):
    """Runs curl and wraps around all necessary hacks."""
    cmd = ['curl']
    cmd.extend(args)

    result = cros_build_lib.RunCommand(cmd, error_ok=True, **kwargs)
    if result.returncode > 0:
      # These are the return codes of failing certs as per 'man curl'.
      if result.returncode in (51, 58, 60):
        print 'Download failed with certificate error? Try "sudo c_rehash".'
      else:
        print 'Curl failed!'
      sys.exit(1)

    return result

  def RemoteTarballExists(url):
    """Tests if a remote tarball exists."""
    result = RunCurl(['-I', url],
                     redirect_stdout=True, redirect_stderr=True,
                     print_cmd=False)
    header = result.output.splitlines()[0]
    return header.find('200 OK') != -1

  url = None
  for url in urls:
    print 'Attempting download: %s' % url
    if RemoteTarballExists(url):
      break
  else:
    raise Exception('No valid URLs found!')

  tarball_name = os.path.basename(urlparse.urlparse(url).path)
  tarball_dest = os.path.join(SDK_DIR, tarball_name)

  # Cleanup old tarballs.
  files_to_delete = [f for f in os.listdir(SDK_DIR) if f != tarball_name]
  if files_to_delete:
    print 'Cleaning up old tarballs: ' + str(files_to_delete)
    for f in files_to_delete:
      f_path = os.path.join(SDK_DIR, f)
      # Only delete regular files that belong to us.
      if os.path.isfile(f_path) and os.stat(f_path).st_uid == os.getuid():
        os.remove(f_path)

  curl_opts = ['-f', '--retry', '5', '-L', '-y', '30',
               '--output', tarball_dest]
  if not url.startswith('file://') and os.path.exists(tarball_dest):
    # Only resume for remote URLs. If the file is local, there's no
    # real speedup, and using the same filename for different files
    # locally will cause issues.
    curl_opts.extend(['-C', '-'])

    # Additionally, certain versions of curl incorrectly fail if
    # told to resume a file that is fully downloaded, thus do a
    # check on our own.
    # see:
    # https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3482927&group_id=976
    result = RunCurl(['-I', url],
                     redirect_stdout=True,
                     redirect_stderr=True,
                     print_cmd=False)

    for x in result.output.splitlines():
      if x.lower().startswith("content-length:"):
        length = int(x.split(":", 1)[-1].strip())
        if length == os.path.getsize(tarball_dest):
          # Fully fetched; bypass invoking curl, since it can screw up handling
          # of this (>=7.21.4 and up).
          return tarball_dest
        break
  curl_opts.append(url)
  RunCurl(curl_opts)
  return tarball_dest

def BootstrapChroot(chroot_path, stage_url, replace):
  """Builds a new chroot from source"""
  cmd = MAKE_CHROOT + ['--chroot', chroot_path,
                       '--nousepkg']

  stage = None
  if stage_url:
    stage = FetchRemoteTarballs([stage_url])

  if stage:
    cmd.extend(['--stage3_path', stage])

  if replace:
    cmd.append('--replace')

  try:
    cros_build_lib.RunCommand(cmd, print_cmd=False)
  except cros_build_lib.RunCommandError:
    raise SystemExit('Running %r failed!' % cmd)

def CreateChroot(sdk_url, sdk_version, chroot_path, replace):
  """Creates a new chroot from a given SDK"""
  if not os.path.exists(SDK_DIR):
    cros_build_lib.RunCommand(['mkdir', '-p', SDK_DIR], print_cmd=False)

  # Based on selections, fetch the tarball
  if sdk_url:
    urls = [sdk_url]
  else:
    arch = GetHostArch()
    urls = GetArchStageTarballs(arch, sdk_version)

  sdk = FetchRemoteTarballs(urls)

  # TODO(zbehan): Unpack and install
  # For now, we simply call make_chroot on the prebuilt chromeos-sdk.
  # make_chroot provides a variety of hacks to make the chroot useable.
  # These should all be eliminated/minimised, after which, we can change
  # this to just unpacking the sdk.
  cmd = MAKE_CHROOT + ['--stage3_path', sdk,
                       '--chroot', chroot_path]

  if replace:
    cmd.append('--replace')

  try:
    cros_build_lib.RunCommand(cmd, print_cmd=False)
  except cros_build_lib.RunCommandError:
    raise SystemExit('Running %r failed!' % cmd)

def DeleteChroot(chroot_path):
  """Deletes an existing chroot"""
  cmd = MAKE_CHROOT + ['--chroot', chroot_path,
                       '--delete']
  try:
    cros_build_lib.RunCommand(cmd, print_cmd=False)
  except cros_build_lib.RunCommandError:
    raise SystemExit('Running %r failed!' % cmd)

def _CreateLockFile(path):
  """Create a lockfile via sudo that is writable by current user."""
  cros_build_lib.SudoRunCommand(['touch', path], print_cmd=False)
  cros_build_lib.SudoRunCommand(['chown', str(os.getuid()), path],
                                print_cmd=False)
  cros_build_lib.SudoRunCommand(['chmod', '644', path], print_cmd=False)

def EnterChroot(chroot_path, chrome_root, chrome_root_mount, additional_args):
  """Enters an existing SDK chroot"""
  cmd = ENTER_CHROOT + ['--chroot', chroot_path]
  if chrome_root:
    cmd.extend(['--chrome_root', chrome_root])
  if chrome_root_mount:
    cmd.extend(['--chrome_root_mount', chrome_root_mount])
  if len(additional_args) > 0:
    cmd.append('--')
    cmd.extend(additional_args)

  ret = cros_build_lib.RunCommand(cmd, print_cmd=False, error_code_ok=True)
  # If we were in interactive mode, ignore the exit code; it'll be whatever
  # they last ran w/in the chroot and won't matter to us one way or another.
  # Note this does allow chroot entrance to fail and be ignored during
  # interactive; this is however a rare case and the user will immediately
  # see it (nor will they be checking the exit code manually).
  if ret.returncode != 0 and additional_args:
    raise SystemExit('Running %r failed with exit code %i'
                     % (cmd, ret.returncode))

def main(argv):
  # TODO(ferringb): make argv required once depot_tools is fixed.
  usage = """usage: %prog [options] [VAR1=val1 .. VARn=valn -- <args>]

This script manages a local CrOS SDK chroot. Depending on the flags,
it can download, build or enter a chroot.

Action taken is the following:
--enter  (default)  .. Installs and enters a chroot
--download          .. Just download a chroot (enter if combined with --enter)
--bootstrap         .. Builds a chroot from source (enter if --enter)
--delete            .. Removes a chroot
"""
  sdk_latest_version = GetLatestVersion()
  parser = optparse.OptionParser(usage)
  # Actions:
  parser.add_option('', '--bootstrap',
                    action='store_true', dest='bootstrap', default=False,
                    help=('Build a new SDK chroot from source'))
  parser.add_option('', '--delete',
                    action='store_true', dest='delete', default=False,
                    help=('Delete the current SDK chroot'))
  parser.add_option('', '--download',
                    action='store_true', dest='download', default=False,
                    help=('Download and install a prebuilt SDK'))
  parser.add_option('', '--enter',
                    action='store_true', dest='enter', default=False,
                    help=('Enter the SDK chroot, possibly (re)create first'))

  # Global options:
  parser.add_option('', '--chroot',
                    dest='chroot', default=constants.DEFAULT_CHROOT_DIR,
                    help=('SDK chroot dir name [%s]' %
                          constants.DEFAULT_CHROOT_DIR))

  # Additional options:
  parser.add_option('', '--chrome_root',
                    dest='chrome_root', default='',
                    help=('Mount this chrome root into the SDK chroot'))
  parser.add_option('', '--chrome_root_mount',
                    dest='chrome_root_mount', default='',
                    help=('Mount chrome into this path inside SDK chroot'))
  parser.add_option('-r', '--replace',
                    action='store_true', dest='replace', default=False,
                    help=('Replace an existing SDK chroot'))
  parser.add_option('-u', '--url',
                    dest='sdk_url', default='',
                    help=('''Use sdk tarball located at this url.
                             Use file:// for local files.'''))
  parser.add_option('-v', '--version',
                    dest='sdk_version', default='',
                    help=('Use this sdk version [%s]' % sdk_latest_version))
  (options, remaining_arguments) = parser.parse_args(argv)

  # Some sanity checks first, before we ask for sudo credentials.
  if cros_build_lib.IsInsideChroot():
    parser.error("This needs to be ran outside the chroot")

  missing = CheckPrerequisites(NEEDED_TOOLS)
  if missing:
    parser.error((
        'The tool(s) %s were not found.'
        'Please install the appropriate package in your host.'
        'Example(ubuntu):'
        '  sudo apt-get install <packagename>'
        % (', '.join(missing))))

  # Default action is --enter, if no other is selected.
  if not (options.bootstrap or options.download or options.delete):
    options.enter = True

  # Only --enter can process additional args as passthrough commands.
  # Warn and exit for least surprise.
  if len(remaining_arguments) > 0 and not options.enter:
    parser.error("Additional arguments are not permitted, unless running "
                 "with --enter")

  # Some actions can be combined, as they merely modify how is the chroot
  # going to be made. The only option that hates all others is --delete.
  if options.delete and \
    (options.enter or options.download or options.bootstrap):
    parser.error("--delete cannot be combined with --enter, "
                 "--download or --bootstrap")
  # NOTE: --delete is a true hater, it doesn't like other options either, but
  # those will hardly lead to confusion. Nobody can expect to pass --version to
  # delete and actually change something.

  if options.bootstrap and options.download:
    parser.error("Either --bootstrap or --download, not both")

  # Bootstrap will start off from a non-selectable stage3 tarball. Attempts to
  # select sdk by version are confusing. Warn and exit. We can still specify a
  # tarball by path or URL though.
  if options.bootstrap and options.sdk_version:
    parser.error("Cannot use --version when bootstrapping")

  chroot_path = os.path.join(SRC_ROOT, options.chroot)
  chroot_path = os.path.abspath(chroot_path)
  chroot_path = os.path.normpath(chroot_path)

  if not options.sdk_version:
    sdk_version = sdk_latest_version
  else:
    sdk_version = options.sdk_version

  if options.delete and not os.path.exists(chroot_path):
    print "Not doing anything. The chroot you want to remove doesn't exist."
    return 0

  lock_path = os.path.dirname(chroot_path)
  lock_path = os.path.join(lock_path,
                           '.%s_lock' % os.path.basename(chroot_path))
  with sudo.SudoKeepAlive():
    with cgroups.SimpleContainChildren('cros_sdk'):
      _CreateLockFile(lock_path)
      with locking.FileLock(lock_path, 'chroot lock') as lock:
        if options.delete:
          lock.write_lock()
          DeleteChroot(chroot_path)
          return 0

        # Print a suggestion for replacement, but not if running just --enter.
        if os.path.exists(chroot_path) and not options.replace and \
            (options.bootstrap or options.download):
          print "Chroot already exists. Run with --replace to re-create."

        # Chroot doesn't exist or asked to replace.
        if not os.path.exists(chroot_path) or options.replace:
          lock.write_lock()
          if options.bootstrap:
            BootstrapChroot(chroot_path, options.sdk_url,
                            options.replace)
          else:
            CreateChroot(options.sdk_url, sdk_version,
                         chroot_path, options.replace)
        if options.enter:
          lock.read_lock()
          EnterChroot(chroot_path, options.chrome_root,
                      options.chrome_root_mount, remaining_arguments)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# coding: utf-8

import os
import pandas as pd

data_directory = "./data/"

"""read in csv's from dapc analisis in R"""
# create a dict of short names and their respective .csv files
fns = {fn.strip().split(".csv")[0]: fn for fn in os.listdir(data_directory) if
       fn in ["assign.csv", "eig.csv", "ind.coord.csv", "posterior.csv", "grp.csv"]}

# create a dict of short names and dataframes sourced from the above .csv files
dfs = {n: pd.read_csv(data_directory + fn) for n, fn in fns.items()}

"""Make key's consistant across all dataframes"""
# for each dataframe
for n, df in dfs.items():
    # rename 0th column to key
    df.rename(columns={df.columns[0]: 'key'}, inplace=True)
    # treat keys as strings even if numeric
    df.key = df.key.apply(str)

"""extract posterior probabilities for a priori group (grp) and model predicted group (assign)"""
# create dataframe with better column names
posterior = dfs["posterior"].rename(columns=lambda x: x.split(".")[-1] if "posterior." in x else x)

# join in the model predicted groups for each individule from the assign column of the assign dataframe
posterior = posterior.join(dfs["assign"]["assign"].apply(str))

# create a 'posterior_assign' column from the column corisponding to each individule's model predicted group
posterior["posterior_assign"] = posterior.apply(lambda row: row.loc[row["assign"]], axis=1)

# join in the a priori groups for each individule from the grp column of the grp dataframe
posterior = posterior.join(dfs["grp"]["grp"].apply(str))

# create a 'posterior_grp' column from the column corisponding to each individule's a priori group
posterior["posterior_grp"] = posterior.apply(lambda row: row.loc[row["grp"]], axis=1)

"""append posterior probabilities to principle components"""
# copy loadings df but with better column names.
df = dfs["ind.coord"].rename(columns=lambda x: x.split(".")[-1] if "ind.coord." in x else x)

# append the 'assign' and 'grp' columns from their respecteve df's as well as the two newly synthisised
# columns created in the 'posterior' df above.
df = df.join([dfs["assign"]["assign"], dfs["grp"]["grp"],
              posterior["posterior_assign"], posterior["posterior_grp"]])

"""append location information"""
# pull location data into a csv
loc_df = pd.read_csv(data_directory + "location.csv")

# be sure to treat key as a string even if it's numeric
loc_df.key = loc_df.key.apply(str)

# join location data to working dataframe 'df'
df = df.merge(loc_df, on="key", how="left")

"""clean up nulls"""
df = df.applymap(lambda x: "NaN" if pd.isnull(x) else x)

"""output"""
df.to_csv(data_directory + "webapp_input.csv", index=False)
<commit_msg>script docstring added, typos corrected<commit_after>#!/usr/bin/env python
# coding: utf-8

"""
This script is used to synthesise a single webapp_input.csv with data of interest pulled from the various dapc tables
as well as from a separately included location.csv

The dapc data includes the keys and loadings from 'ind.coord.csv'; the a priori group (grp) and
model predicted group (assign) from their respective csv's; and the corresponding posterior probabilities
for the 'grp' and 'assign' columns from 'posterior.csv'.

The location.csv must include at least a 'key', 'lat', and 'lng' column, but can include any additional metadata.
"""

import os
import pandas as pd

data_directory = "./data/"

"""read in csv's from dapc analysis in R"""
# create a dict of short names and their respective .csv files
fns = {fn.strip().split(".csv")[0]: fn for fn in os.listdir(data_directory) if
       fn in ["assign.csv", "eig.csv", "ind.coord.csv", "posterior.csv", "grp.csv"]}

# create a dict of short names and dataframes sourced from the above .csv files
dfs = {n: pd.read_csv(data_directory + fn) for n, fn in fns.items()}

"""Make key's consistent across all dataframes"""
# for each dataframe
for n, df in dfs.items():
    # rename 0th column to key
    df.rename(columns={df.columns[0]: 'key'}, inplace=True)
    # treat keys as strings even if numeric
    df.key = df.key.apply(str)

"""extract posterior probabilities for a priori group (grp) and model predicted group (assign)"""
# create dataframe with better column names
posterior = dfs["posterior"].rename(columns=lambda x: x.split(".")[-1] if "posterior." in x else x)

# join in the model predicted groups for each individual from the assign column of the assign dataframe
posterior = posterior.join(dfs["assign"]["assign"].apply(str))

# create a 'posterior_assign' column from the column corresponding to each individual's model predicted group
posterior["posterior_assign"] = posterior.apply(lambda row: row.loc[row["assign"]], axis=1)

# join in the a priori groups for each individual from the grp column of the grp dataframe
posterior = posterior.join(dfs["grp"]["grp"].apply(str))

# create a 'posterior_grp' column from the column corresponding to each individual's a priori group
posterior["posterior_grp"] = posterior.apply(lambda row: row.loc[row["grp"]], axis=1)

"""append posterior probabilities to principle components"""
# copy loadings df but with better column names.
df = dfs["ind.coord"].rename(columns=lambda x: x.split(".")[-1] if "ind.coord." in x else x)

# append the 'assign' and 'grp' columns from their respective df's as well as the two newly synthesised
# columns created in the 'posterior' df above.
df = df.join([dfs["assign"]["assign"], dfs["grp"]["grp"],
              posterior["posterior_assign"], posterior["posterior_grp"]])

"""append location information"""
# pull location data into a csv
loc_df = pd.read_csv(data_directory + "location.csv")

# be sure to treat key as a string even if it's numeric
loc_df.key = loc_df.key.apply(str)

# join location data to working dataframe 'df'
df = df.merge(loc_df, on="key", how="left")

"""clean up nulls"""
df = df.applymap(lambda x: "NaN" if pd.isnull(x) else x)

"""output"""
df.to_csv(data_directory + "webapp_input.csv", index=False)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
import collections
import re

import click
import dataproperty
import path
import simplesqlite
from simplesqlite.loader import ValidationError
from simplesqlite.loader import InvalidDataError

CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])

class LoaderNotFound(Exception):
    pass

class LoaderFactory(object):
    LoaderTuple = collections.namedtuple(
        "LoaderTuple", "filename_regexp loader")

    LOADERTUPLE_LIST = [
        LoaderTuple(
            re.compile("[\.]csv$"),
            simplesqlite.loader.CsvTableFileLoader()),
        LoaderTuple(
            re.compile("[\.]json$"),
            simplesqlite.loader.JsonTableFileLoader()),
        LoaderTuple(
            re.compile("[\.]xlsx$|[\.]xlsm$|[\.]xls$"),
            simplesqlite.loader.ExcelTableFileLoader()),
    ]

    @classmethod
    def get_loader(cls, file_path):
        for loadertuple in cls.LOADERTUPLE_LIST:
            if loadertuple.filename_regexp.search(file_path) is None:
                continue

            loadertuple.loader.source = file_path

            return loadertuple.loader

        raise LoaderNotFound(file_path)

def create_database(database_path):
    db_path = path.Path(database_path)
    dir_path = db_path.dirname()
    if dataproperty.is_not_empty_string(dir_path):
        dir_path.makedirs_p()

    return simplesqlite.SimpleSQLite(db_path, "w")

@click.group(context_settings=CONTEXT_SETTINGS)
@click.version_option()
def cmd():
    pass

@cmd.command()
@click.argument("files", type=str, nargs=-1)
@click.option(
    "-o", "--output-path", default="out.sqlite",
    help="Output path of the SQLite database file")
def file(files, output_path):
    """
    Convert CSV/JSON/Excel file(s) to a SQLite database file.
    """

    con = create_database(output_path)

    convert_count = 0
    for file_path in files:
        if not path.Path(file_path).isfile():
            continue

        try:
            loader = LoaderFactory.get_loader(file_path)
        except LoaderNotFound:
            continue

        for tabledata in loader.load():
            click.echo("convert '{:s}' to '{:s}' table".format(
                file_path, tabledata.table_name))

            try:
                con.create_table_from_tabledata(tabledata)
                convert_count += 1
            except (ValueError, IOError, ValidationError, InvalidDataError):
                continue

    return 0 if convert_count == 0 else 1

@cmd.command()
@click.argument(
    "credentials", type=click.Path(exists=True))
@click.argument(
    "title", type=str)
@click.option(
    "-o", "--output-path", default="out.sqlite",
    help="output path of the SQLite database file")
def gs(credentials, title, output_path):
    """
    Convert Google Sheets to a SQLite database file.

    CREDENTIALS: OAuth2 Google credentials file.
    TITLE: Title of the Google Sheets to convert.
    """

    con = create_database(output_path)

    loader = simplesqlite.loader.GoogleSheetsTableLoader()
    loader.source = credentials
    loader.title = title

    for tabledata in loader.load():
        click.echo("convert '{:s}' to '{:s}' table".format(
            title, tabledata.table_name))
        con.create_table_from_tabledata(tabledata)

    return 0

if __name__ == '__main__':
    cmd()
<commit_msg>Fix exit code<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
import collections
import re
import sys

import click
import dataproperty
import path
import simplesqlite
from simplesqlite.loader import ValidationError
from simplesqlite.loader import InvalidDataError

from ._counter import ResultCounter

CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])

class LoaderNotFound(Exception):
    pass

class LoaderFactory(object):
    LoaderTuple = collections.namedtuple(
        "LoaderTuple", "filename_regexp loader")

    LOADERTUPLE_LIST = [
        LoaderTuple(
            re.compile("[\.]csv$"),
            simplesqlite.loader.CsvTableFileLoader()),
        LoaderTuple(
            re.compile("[\.]json$"),
            simplesqlite.loader.JsonTableFileLoader()),
        LoaderTuple(
            re.compile("[\.]xlsx$|[\.]xlsm$|[\.]xls$"),
            simplesqlite.loader.ExcelTableFileLoader()),
    ]

    @classmethod
    def get_loader(cls, file_path):
        for loadertuple in cls.LOADERTUPLE_LIST:
            if loadertuple.filename_regexp.search(file_path) is None:
                continue

            loadertuple.loader.source = file_path

            return loadertuple.loader

        raise LoaderNotFound(file_path)

def create_database(database_path):
    db_path = path.Path(database_path)
    dir_path = db_path.dirname()
    if dataproperty.is_not_empty_string(dir_path):
        dir_path.makedirs_p()

    return simplesqlite.SimpleSQLite(db_path, "w")

@click.group(context_settings=CONTEXT_SETTINGS)
@click.version_option()
def cmd():
    pass

@cmd.command()
@click.argument("files", type=str, nargs=-1)
@click.option(
    "-o", "--output-path", default="out.sqlite",
    help="Output path of the SQLite database file")
def file(files, output_path):
    """
    Convert CSV/JSON/Excel file(s) to a SQLite database file.
    """

    con = create_database(output_path)
    result_counter = ResultCounter()

    for file_path in files:
        if not path.Path(file_path).isfile():
            continue

        try:
            loader = LoaderFactory.get_loader(file_path)
        except LoaderNotFound:
            continue

        try:
            for tabledata in loader.load():
                click.echo("convert '{:s}' to '{:s}' table".format(
                    file_path, tabledata.table_name))

                try:
                    con.create_table_from_tabledata(tabledata)
                    result_counter.inc_success()
                except (ValueError, IOError):
                    result_counter.inc_fail()
                    continue
        except (ValidationError, InvalidDataError):
            result_counter.inc_fail()

    sys.exit(result_counter.get_return_code())

@cmd.command()
@click.argument(
    "credentials", type=click.Path(exists=True))
@click.argument(
    "title", type=str)
@click.option(
    "-o", "--output-path", default="out.sqlite",
    help="output path of the SQLite database file")
def gs(credentials, title, output_path):
    """
    Convert Google Sheets to a SQLite database file.

    CREDENTIALS: OAuth2 Google credentials file.
    TITLE: Title of the Google Sheets to convert.
    """

    con = create_database(output_path)
    result_counter = ResultCounter()

    loader = simplesqlite.loader.GoogleSheetsTableLoader()
    loader.source = credentials
    loader.title = title

    try:
        for tabledata in loader.load():
            click.echo("convert '{:s}' to '{:s}' table".format(
                title, tabledata.table_name))

            try:
                con.create_table_from_tabledata(tabledata)
                result_counter.inc_success()
            except (ValidationError, InvalidDataError):
                result_counter.inc_fail()
    except (ValidationError, InvalidDataError):
        result_counter.inc_fail()

    sys.exit(result_counter.get_return_code())

if __name__ == '__main__':
    cmd()
<|endoftext|>