<commit_before>#
# Small ftp server for ESP8266 ans ESP32 Micropython
#
# Based on the work of chrisgp - Christopher Popp and pfalcon - Paul Sokolovsky
#
# The server accepts passive mode only.
# It runs in foreground and quits, when it receives a quit command
# Start the server with:
#
# import ftp
#
# Copyright (c) 2016 Christopher Popp (initial ftp server framework)
# Copyright (c) 2016 Robert Hammelrath (putting the pieces together and a few extensions)
# Distributed under MIT License
#
import socket
import network
import uos
import gc

def send_list_data(path, dataclient, full):
    try: # whether path is a directory name
        for fname in sorted(uos.listdir(path), key = str.lower):
            dataclient.sendall(make_description(path, fname, full))
    except: # path may be a file name or pattern
        pattern = path.split("/")[-1]
        path = path[:-(len(pattern) + 1)]
        if path == "": path = "/"
        for fname in sorted(uos.listdir(path), key = str.lower):
            if fncmp(fname, pattern) == True:
                dataclient.sendall(make_description(path, fname, full))

def make_description(path, fname, full):
    if full:
        stat = uos.stat(get_absolute_path(path,fname))
        file_permissions = "drwxr-xr-x" if (stat[0] & 0o170000 == 0o040000) else "-rw-r--r--"
        file_size = stat[6]
        description = "{}    1 owner group {:>10} Jan 1 2000 {}\r\n".format(
                file_permissions, file_size, fname)
    else:
        description = fname + "\r\n"
    return description

def send_file_data(path, dataclient):
    with open(path, "r") as file:
        chunk = file.read(512)
        while len(chunk) > 0:
            dataclient.sendall(chunk)
            chunk = file.read(512)

def save_file_data(path, dataclient):
    with open(path, "w") as file:
        chunk = dataclient.recv(512)
        while len(chunk) > 0:
            file.write(chunk)
            chunk = dataclient.recv(512)

def get_absolute_path(cwd, payload):
    # Just a few special cases "..", "." and ""
    # If payload start's with /, set cwd to /
    # and consider the remainder a relative path
    if payload.startswith('/'):
        cwd = "/"
    for token in payload.split("/"):
        if token == '..':
            if cwd != '/':
                cwd = '/'.join(cwd.split('/')[:-1])
                if cwd == '':
                    cwd = '/'
        elif token != '.' and token != '':
            if cwd == '/':
                cwd += token
            else:
                cwd = cwd + '/' + token
    return cwd

# compare fname against pattern. Pattern may contain
# wildcards ? and *.
def fncmp(fname, pattern):
    pi = 0
    si = 0
    while pi < len(pattern) and si < len(fname):
        if (fname[si] == pattern[pi]) or (pattern[pi] == '?'):
            si += 1
            pi += 1
        else:
            if pattern[pi] == '*': # recurse
                if (pi + 1) == len(pattern):
                    return True
                while si < len(fname):
                    if fncmp(fname[si:], pattern[pi+1:]) == True:
                        return True
                    else:
                        si += 1
                return False
            else:
                return False
    if pi == len(pattern.rstrip("*"))  and si == len(fname):
        return True
    else:
        return False

def ftpserver():

    DATA_PORT = 13333

    ftpsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    datasocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    ftpsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    datasocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    ftpsocket.bind(socket.getaddrinfo("0.0.0.0", 21)[0][4])
    datasocket.bind(socket.getaddrinfo("0.0.0.0", DATA_PORT)[0][4])

    ftpsocket.listen(1)
    ftpsocket.settimeout(None)
    datasocket.listen(1)
    datasocket.settimeout(None)

    msg_250_OK = '250 OK\r\n'
    msg_550_fail = '550 Failed\r\n'
    addr = network.WLAN().ifconfig()[0]
    print("FTP Server started on ", addr)
    try:
        dataclient = None
        fromname = None
        do_run = True
        while do_run:
            cl, remote_addr = ftpsocket.accept()
            cl.settimeout(300)
            cwd = '/'
            try:
                # print("FTP connection from:", remote_addr)
                cl.sendall("220 Hello, this is the ESP8266/ESP32.\r\n")
                while True:
                    gc.collect()
                    data = cl.readline().decode("utf-8").rstrip("\r\n")
                    if len(data) <= 0:
                        print("Client disappeared")
                        do_run = False
                        break

                    command = data.split(" ")[0].upper()
                    payload = data[len(command):].lstrip()

                    path = get_absolute_path(cwd, payload)

                    print("Command={}, Payload={}".format(command, payload))

                    if command == "USER":
                        cl.sendall("230 Logged in.\r\n")
                    elif command == "SYST":
                        cl.sendall("215 UNIX Type: L8\r\n")
                    elif command == "NOOP":
                        cl.sendall("200 OK\r\n")
                    elif command == "FEAT":
                        cl.sendall("211 no-features\r\n")
                    elif command == "PWD":
                        cl.sendall('257 "{}"\r\n'.format(cwd))
                    elif command == "CWD":
                        try:
                            files = uos.listdir(path)
                            cwd = path
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "CDUP":
                        cwd = get_absolute_path(cwd, "..")
                        cl.sendall(msg_250_OK)
                    elif command == "TYPE":
                        # probably should switch between binary and not
                        cl.sendall('200 Transfer mode set\r\n')
                    elif command == "SIZE":
                        try:
                            size = uos.stat(path)[6]
                            cl.sendall('213 {}\r\n'.format(size))
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "QUIT":
                        cl.sendall('221 Bye.\r\n')
                        do_run = False
                        break
                    elif command == "PASV":
                        cl.sendall('227 Entering Passive Mode ({},{},{}).\r\n'.format(
                            addr.replace('.',','), DATA_PORT>>8, DATA_PORT%256))
                        dataclient, data_addr = datasocket.accept()
                        print("FTP Data connection from:", data_addr)
                    elif command == "LIST" or command == "NLST":
                        if not payload.startswith("-"):
                            place = path
                        else:
                            place = cwd
                        try:
                            send_list_data(place, dataclient, command == "LIST" or payload == "-l")
                            cl.sendall("150 Here comes the directory listing.\r\n")
                            cl.sendall("226 Listed.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "RETR":
                        try:
                            send_file_data(path, dataclient)
                            cl.sendall("150 Opening data connection.\r\n")
                            cl.sendall("226 Transfer complete.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "STOR":
                        try:
                            cl.sendall("150 Ok to send data.\r\n")
                            save_file_data(path, dataclient)
                            cl.sendall("226 Transfer complete.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "DELE":
                        try:
                            uos.remove(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "RMD":
                        try:
                            uos.rmdir(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "MKD":
                        try:
                            uos.mkdir(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "RNFR":
                            fromname = path
                            cl.sendall("350 Rename from\r\n")
                    elif command == "RNTO":
                            if fromname is not None:
                                try:
                                    uos.rename(fromname, path)
                                    cl.sendall(msg_250_OK)
                                except:
                                    cl.sendall(msg_550_fail)
                            else:
                                cl.sendall(msg_550_fail)
                            fromname = None
                    elif command == "STAT":
                        if payload == "":
                            cl.sendall("211-Connected to ({})\r\n"
                                       "    Data address ({})\r\n"
                                       "211 TYPE: Binary STRU: File MODE: Stream\r\n".format(
                                       remote_addr[0], addr))
                        else:
                            cl.sendall("213-Directory listing:\r\n")
                            send_list_data(path, cl, True)
                            cl.sendall("213 Done.\r\n")
                    else:
                        cl.sendall("502 Unsupported command.\r\n")
                        print("Unsupported command {} with payload {}".format(command, payload))
            except Exception as err:
                print(err)

            finally:
                cl.close()
                cl = None
    finally:
        datasocket.close()
        ftpsocket.close()
        if dataclient is not None:
            dataclient.close()

ftpserver()
<commit_msg>ftp.py: Support connection in AP mode too<commit_after>#
# Small ftp server for ESP8266 ans ESP32 Micropython
#
# Based on the work of chrisgp - Christopher Popp and pfalcon - Paul Sokolovsky
#
# The server accepts passive mode only.
# It runs in foreground and quits, when it receives a quit command
# Start the server with:
#
# import ftp
#
# Copyright (c) 2016 Christopher Popp (initial ftp server framework)
# Copyright (c) 2016 Robert Hammelrath (putting the pieces together and a few extensions)
# Distributed under MIT License
#
import socket
import network
import uos
import gc

def send_list_data(path, dataclient, full):
    try: # whether path is a directory name
        for fname in sorted(uos.listdir(path), key = str.lower):
            dataclient.sendall(make_description(path, fname, full))
    except: # path may be a file name or pattern
        pattern = path.split("/")[-1]
        path = path[:-(len(pattern) + 1)]
        if path == "": path = "/"
        for fname in sorted(uos.listdir(path), key = str.lower):
            if fncmp(fname, pattern) == True:
                dataclient.sendall(make_description(path, fname, full))

def make_description(path, fname, full):
    if full:
        stat = uos.stat(get_absolute_path(path,fname))
        file_permissions = "drwxr-xr-x" if (stat[0] & 0o170000 == 0o040000) else "-rw-r--r--"
        file_size = stat[6]
        description = "{}    1 owner group {:>10} Jan 1 2000 {}\r\n".format(
                file_permissions, file_size, fname)
    else:
        description = fname + "\r\n"
    return description

def send_file_data(path, dataclient):
    with open(path, "r") as file:
        chunk = file.read(512)
        while len(chunk) > 0:
            dataclient.sendall(chunk)
            chunk = file.read(512)

def save_file_data(path, dataclient):
    with open(path, "w") as file:
        chunk = dataclient.recv(512)
        while len(chunk) > 0:
            file.write(chunk)
            chunk = dataclient.recv(512)

def get_absolute_path(cwd, payload):
    # Just a few special cases "..", "." and ""
    # If payload start's with /, set cwd to /
    # and consider the remainder a relative path
    if payload.startswith('/'):
        cwd = "/"
    for token in payload.split("/"):
        if token == '..':
            if cwd != '/':
                cwd = '/'.join(cwd.split('/')[:-1])
                if cwd == '':
                    cwd = '/'
        elif token != '.' and token != '':
            if cwd == '/':
                cwd += token
            else:
                cwd = cwd + '/' + token
    return cwd

# compare fname against pattern. Pattern may contain
# wildcards ? and *.
def fncmp(fname, pattern):
    pi = 0
    si = 0
    while pi < len(pattern) and si < len(fname):
        if (fname[si] == pattern[pi]) or (pattern[pi] == '?'):
            si += 1
            pi += 1
        else:
            if pattern[pi] == '*': # recurse
                if (pi + 1) == len(pattern):
                    return True
                while si < len(fname):
                    if fncmp(fname[si:], pattern[pi+1:]) == True:
                        return True
                    else:
                        si += 1
                return False
            else:
                return False
    if pi == len(pattern.rstrip("*"))  and si == len(fname):
        return True
    else:
        return False

def ftpserver():

    DATA_PORT = 13333

    ftpsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    datasocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    ftpsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    datasocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    ftpsocket.bind(socket.getaddrinfo("0.0.0.0", 21)[0][4])
    datasocket.bind(socket.getaddrinfo("0.0.0.0", DATA_PORT)[0][4])

    ftpsocket.listen(1)
    ftpsocket.settimeout(None)
    datasocket.listen(1)
    datasocket.settimeout(None)

    msg_250_OK = '250 OK\r\n'
    msg_550_fail = '550 Failed\r\n'
    wlan = network.WLAN(network.AP_IF)
    if wlan.active():
        ifconfig = wlan.ifconfig()
    else:
        wlan = network.WLAN(network.STA_IF)
        if wlan.active():
            ifconfig = wlan.ifconfig()
        else:
            print("No active connection")
            return
    addr = ifconfig[0]
    print("FTP Server started on ", addr)
    try:
        dataclient = None
        fromname = None
        do_run = True
        while do_run:
            cl, remote_addr = ftpsocket.accept()
            cl.settimeout(300)
            cwd = '/'
            try:
                # print("FTP connection from:", remote_addr)
                cl.sendall("220 Hello, this is the ESP8266/ESP32.\r\n")
                while True:
                    gc.collect()
                    data = cl.readline().decode("utf-8").rstrip("\r\n")
                    if len(data) <= 0:
                        print("Client disappeared")
                        do_run = False
                        break

                    command = data.split(" ")[0].upper()
                    payload = data[len(command):].lstrip()

                    path = get_absolute_path(cwd, payload)

                    print("Command={}, Payload={}".format(command, payload))

                    if command == "USER":
                        cl.sendall("230 Logged in.\r\n")
                    elif command == "SYST":
                        cl.sendall("215 UNIX Type: L8\r\n")
                    elif command == "NOOP":
                        cl.sendall("200 OK\r\n")
                    elif command == "FEAT":
                        cl.sendall("211 no-features\r\n")
                    elif command == "PWD":
                        cl.sendall('257 "{}"\r\n'.format(cwd))
                    elif command == "CWD":
                        try:
                            files = uos.listdir(path)
                            cwd = path
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "CDUP":
                        cwd = get_absolute_path(cwd, "..")
                        cl.sendall(msg_250_OK)
                    elif command == "TYPE":
                        # probably should switch between binary and not
                        cl.sendall('200 Transfer mode set\r\n')
                    elif command == "SIZE":
                        try:
                            size = uos.stat(path)[6]
                            cl.sendall('213 {}\r\n'.format(size))
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "QUIT":
                        cl.sendall('221 Bye.\r\n')
                        do_run = False
                        break
                    elif command == "PASV":
                        cl.sendall('227 Entering Passive Mode ({},{},{}).\r\n'.format(
                            addr.replace('.',','), DATA_PORT>>8, DATA_PORT%256))
                        dataclient, data_addr = datasocket.accept()
                        print("FTP Data connection from:", data_addr)
                    elif command == "LIST" or command == "NLST":
                        if not payload.startswith("-"):
                            place = path
                        else:
                            place = cwd
                        try:
                            send_list_data(place, dataclient, command == "LIST" or payload == "-l")
                            cl.sendall("150 Here comes the directory listing.\r\n")
                            cl.sendall("226 Listed.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "RETR":
                        try:
                            send_file_data(path, dataclient)
                            cl.sendall("150 Opening data connection.\r\n")
                            cl.sendall("226 Transfer complete.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "STOR":
                        try:
                            cl.sendall("150 Ok to send data.\r\n")
                            save_file_data(path, dataclient)
                            cl.sendall("226 Transfer complete.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "DELE":
                        try:
                            uos.remove(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "RMD":
                        try:
                            uos.rmdir(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "MKD":
                        try:
                            uos.mkdir(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "RNFR":
                            fromname = path
                            cl.sendall("350 Rename from\r\n")
                    elif command == "RNTO":
                            if fromname is not None:
                                try:
                                    uos.rename(fromname, path)
                                    cl.sendall(msg_250_OK)
                                except:
                                    cl.sendall(msg_550_fail)
                            else:
                                cl.sendall(msg_550_fail)
                            fromname = None
                    elif command == "STAT":
                        if payload == "":
                            cl.sendall("211-Connected to ({})\r\n"
                                       "    Data address ({})\r\n"
                                       "211 TYPE: Binary STRU: File MODE: Stream\r\n".format(
                                       remote_addr[0], addr))
                        else:
                            cl.sendall("213-Directory listing:\r\n")
                            send_list_data(path, cl, True)
                            cl.sendall("213 Done.\r\n")
                    else:
                        cl.sendall("502 Unsupported command.\r\n")
                        print("Unsupported command {} with payload {}".format(command, payload))
            except Exception as err:
                print(err)

            finally:
                cl.close()
                cl = None
    finally:
        datasocket.close()
        ftpsocket.close()
        if dataclient is not None:
            dataclient.close()

ftpserver()
<|endoftext|>


<commit_before>#
# Small ftp server for ESP8266 ans ESP32 Micropython
#
# Based on the work of chrisgp - Christopher Popp and pfalcon - Paul Sokolovsky
#
# The server accepts passive mode only.
# It runs in foreground and quits, when it receives a quit command
# Start the server with:
#
# import ftp
#
# Copyright (c) 2016 Christopher Popp (initial ftp server framework)
# Copyright (c) 2016 Robert Hammelrath (putting the pieces together and a few extensions)
# Distributed under MIT License
#
import socket
import network
import uos
import gc

def send_list_data(path, dataclient, full):
    try: # whether path is a directory name
        for fname in sorted(uos.listdir(path), key = str.lower):
            dataclient.sendall(make_description(path, fname, full))
    except: # path may be a file name or pattern
        pattern = path.split("/")[-1]
        path = path[:-(len(pattern) + 1)]
        if path == "": path = "/"
        for fname in sorted(uos.listdir(path), key = str.lower):
            if fncmp(fname, pattern) == True:
                dataclient.sendall(make_description(path, fname, full))

def make_description(path, fname, full):
    if full:
        stat = uos.stat(get_absolute_path(path,fname))
        file_permissions = "drwxr-xr-x" if (stat[0] & 0o170000 == 0o040000) else "-rw-r--r--"
        file_size = stat[6]
        description = "{}    1 owner group {:>10} Jan 1 2000 {}\r\n".format(
                file_permissions, file_size, fname)
    else:
        description = fname + "\r\n"
    return description

def send_file_data(path, dataclient):
    with open(path, "r") as file:
        chunk = file.read(512)
        while len(chunk) > 0:
            dataclient.sendall(chunk)
            chunk = file.read(512)

def save_file_data(path, dataclient):
    with open(path, "w") as file:
        chunk = dataclient.recv(512)
        while len(chunk) > 0:
            file.write(chunk)
            chunk = dataclient.recv(512)

def get_absolute_path(cwd, payload):
    # Just a few special cases "..", "." and ""
    # If payload start's with /, set cwd to /
    # and consider the remainder a relative path
    if payload.startswith('/'):
        cwd = "/"
    for token in payload.split("/"):
        if token == '..':
            if cwd != '/':
                cwd = '/'.join(cwd.split('/')[:-1])
                if cwd == '':
                    cwd = '/'
        elif token != '.' and token != '':
            if cwd == '/':
                cwd += token
            else:
                cwd = cwd + '/' + token
    return cwd

# compare fname against pattern. Pattern may contain
# wildcards ? and *.
def fncmp(fname, pattern):
    pi = 0
    si = 0
    while pi < len(pattern) and si < len(fname):
        if (fname[si] == pattern[pi]) or (pattern[pi] == '?'):
            si += 1
            pi += 1
        else:
            if pattern[pi] == '*': # recurse
                if (pi + 1) == len(pattern):
                    return True
                while si < len(fname):
                    if fncmp(fname[si:], pattern[pi+1:]) == True:
                        return True
                    else:
                        si += 1
                return False
            else:
                return False
    if pi == len(pattern.rstrip("*"))  and si == len(fname):
        return True
    else:
        return False

def ftpserver():

    DATA_PORT = 13333

    ftpsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    datasocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    ftpsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    datasocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    ftpsocket.bind(socket.getaddrinfo("0.0.0.0", 21)[0][4])
    datasocket.bind(socket.getaddrinfo("0.0.0.0", DATA_PORT)[0][4])

    ftpsocket.listen(1)
    ftpsocket.settimeout(None)
    datasocket.listen(1)
    datasocket.settimeout(None)

    msg_250_OK = '250 OK\r\n'
    msg_550_fail = '550 Failed\r\n'
    wlan = network.WLAN(network.AP_IF)
    if wlan.active():
        ifconfig = wlan.ifconfig()
    else:
        wlan = network.WLAN(network.STA_IF)
        if wlan.active():
            ifconfig = wlan.ifconfig()
        else:
            print("No active connection")
            return
    addr = ifconfig[0]
    print("FTP Server started on ", addr)
    try:
        dataclient = None
        fromname = None
        do_run = True
        while do_run:
            cl, remote_addr = ftpsocket.accept()
            cl.settimeout(300)
            cwd = '/'
            try:
                # print("FTP connection from:", remote_addr)
                cl.sendall("220 Hello, this is the ESP8266/ESP32.\r\n")
                while True:
                    gc.collect()
                    data = cl.readline().decode("utf-8").rstrip("\r\n")
                    if len(data) <= 0:
                        print("Client disappeared")
                        do_run = False
                        break

                    command = data.split(" ")[0].upper()
                    payload = data[len(command):].lstrip()

                    path = get_absolute_path(cwd, payload)

                    print("Command={}, Payload={}".format(command, payload))

                    if command == "USER":
                        cl.sendall("230 Logged in.\r\n")
                    elif command == "SYST":
                        cl.sendall("215 UNIX Type: L8\r\n")
                    elif command == "NOOP":
                        cl.sendall("200 OK\r\n")
                    elif command == "FEAT":
                        cl.sendall("211 no-features\r\n")
                    elif command == "PWD":
                        cl.sendall('257 "{}"\r\n'.format(cwd))
                    elif command == "CWD":
                        try:
                            files = uos.listdir(path)
                            cwd = path
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "CDUP":
                        cwd = get_absolute_path(cwd, "..")
                        cl.sendall(msg_250_OK)
                    elif command == "TYPE":
                        # probably should switch between binary and not
                        cl.sendall('200 Transfer mode set\r\n')
                    elif command == "SIZE":
                        try:
                            size = uos.stat(path)[6]
                            cl.sendall('213 {}\r\n'.format(size))
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "QUIT":
                        cl.sendall('221 Bye.\r\n')
                        do_run = False
                        break
                    elif command == "PASV":
                        cl.sendall('227 Entering Passive Mode ({},{},{}).\r\n'.format(
                            addr.replace('.',','), DATA_PORT>>8, DATA_PORT%256))
                        dataclient, data_addr = datasocket.accept()
                        print("FTP Data connection from:", data_addr)
                    elif command == "LIST" or command == "NLST":
                        if not payload.startswith("-"):
                            place = path
                        else:
                            place = cwd
                        try:
                            send_list_data(place, dataclient, command == "LIST" or payload == "-l")
                            cl.sendall("150 Here comes the directory listing.\r\n")
                            cl.sendall("226 Listed.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "RETR":
                        try:
                            send_file_data(path, dataclient)
                            cl.sendall("150 Opening data connection.\r\n")
                            cl.sendall("226 Transfer complete.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "STOR":
                        try:
                            cl.sendall("150 Ok to send data.\r\n")
                            save_file_data(path, dataclient)
                            cl.sendall("226 Transfer complete.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "DELE":
                        try:
                            uos.remove(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "RMD":
                        try:
                            uos.rmdir(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "MKD":
                        try:
                            uos.mkdir(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "RNFR":
                            fromname = path
                            cl.sendall("350 Rename from\r\n")
                    elif command == "RNTO":
                            if fromname is not None:
                                try:
                                    uos.rename(fromname, path)
                                    cl.sendall(msg_250_OK)
                                except:
                                    cl.sendall(msg_550_fail)
                            else:
                                cl.sendall(msg_550_fail)
                            fromname = None
                    elif command == "STAT":
                        if payload == "":
                            cl.sendall("211-Connected to ({})\r\n"
                                       "    Data address ({})\r\n"
                                       "211 TYPE: Binary STRU: File MODE: Stream\r\n".format(
                                       remote_addr[0], addr))
                        else:
                            cl.sendall("213-Directory listing:\r\n")
                            send_list_data(path, cl, True)
                            cl.sendall("213 Done.\r\n")
                    else:
                        cl.sendall("502 Unsupported command.\r\n")
                        print("Unsupported command {} with payload {}".format(command, payload))
            except Exception as err:
                print(err)

            finally:
                cl.close()
                cl = None
    finally:
        datasocket.close()
        ftpsocket.close()
        if dataclient is not None:
            dataclient.close()

ftpserver()
<commit_msg>ftp.py: Slighrt reordering or statements for LIST and RECV<commit_after>#
# Small ftp server for ESP8266 ans ESP32 Micropython
#
# Based on the work of chrisgp - Christopher Popp and pfalcon - Paul Sokolovsky
#
# The server accepts passive mode only.
# It runs in foreground and quits, when it receives a quit command
# Start the server with:
#
# import ftp
#
# Copyright (c) 2016 Christopher Popp (initial ftp server framework)
# Copyright (c) 2016 Robert Hammelrath (putting the pieces together and a few extensions)
# Distributed under MIT License
#
import socket
import network
import uos
import gc

def send_list_data(path, dataclient, full):
    try: # whether path is a directory name
        for fname in sorted(uos.listdir(path), key = str.lower):
            dataclient.sendall(make_description(path, fname, full))
    except: # path may be a file name or pattern
        pattern = path.split("/")[-1]
        path = path[:-(len(pattern) + 1)]
        if path == "": path = "/"
        for fname in sorted(uos.listdir(path), key = str.lower):
            if fncmp(fname, pattern) == True:
                dataclient.sendall(make_description(path, fname, full))

def make_description(path, fname, full):
    if full:
        stat = uos.stat(get_absolute_path(path,fname))
        file_permissions = "drwxr-xr-x" if (stat[0] & 0o170000 == 0o040000) else "-rw-r--r--"
        file_size = stat[6]
        description = "{}    1 owner group {:>10} Jan 1 2000 {}\r\n".format(
                file_permissions, file_size, fname)
    else:
        description = fname + "\r\n"
    return description

def send_file_data(path, dataclient):
    with open(path, "r") as file:
        chunk = file.read(512)
        while len(chunk) > 0:
            dataclient.sendall(chunk)
            chunk = file.read(512)

def save_file_data(path, dataclient):
    with open(path, "w") as file:
        chunk = dataclient.recv(512)
        while len(chunk) > 0:
            file.write(chunk)
            chunk = dataclient.recv(512)

def get_absolute_path(cwd, payload):
    # Just a few special cases "..", "." and ""
    # If payload start's with /, set cwd to /
    # and consider the remainder a relative path
    if payload.startswith('/'):
        cwd = "/"
    for token in payload.split("/"):
        if token == '..':
            if cwd != '/':
                cwd = '/'.join(cwd.split('/')[:-1])
                if cwd == '':
                    cwd = '/'
        elif token != '.' and token != '':
            if cwd == '/':
                cwd += token
            else:
                cwd = cwd + '/' + token
    return cwd

# compare fname against pattern. Pattern may contain
# wildcards ? and *.
def fncmp(fname, pattern):
    pi = 0
    si = 0
    while pi < len(pattern) and si < len(fname):
        if (fname[si] == pattern[pi]) or (pattern[pi] == '?'):
            si += 1
            pi += 1
        else:
            if pattern[pi] == '*': # recurse
                if (pi + 1) == len(pattern):
                    return True
                while si < len(fname):
                    if fncmp(fname[si:], pattern[pi+1:]) == True:
                        return True
                    else:
                        si += 1
                return False
            else:
                return False
    if pi == len(pattern.rstrip("*"))  and si == len(fname):
        return True
    else:
        return False

def ftpserver():

    DATA_PORT = 13333

    ftpsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    datasocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    ftpsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    datasocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    ftpsocket.bind(socket.getaddrinfo("0.0.0.0", 21)[0][4])
    datasocket.bind(socket.getaddrinfo("0.0.0.0", DATA_PORT)[0][4])

    ftpsocket.listen(1)
    ftpsocket.settimeout(None)
    datasocket.listen(1)
    datasocket.settimeout(None)

    msg_250_OK = '250 OK\r\n'
    msg_550_fail = '550 Failed\r\n'
    # check for an active interface, STA first
    wlan = network.WLAN(network.STA_IF)
    if wlan.active():
        addr = wlan.ifconfig()[0]
    else:
        wlan = network.WLAN(network.AP_IF)
        if wlan.active():
            addr = wlan.ifconfig()[0]
        else:
            print("No active connection")
            return

    print("FTP Server started on ", addr)
    try:
        dataclient = None
        fromname = None
        do_run = True
        while do_run:
            cl, remote_addr = ftpsocket.accept()
            cl.settimeout(300)
            cwd = '/'
            try:
                # print("FTP connection from:", remote_addr)
                cl.sendall("220 Hello, this is the ESP8266/ESP32.\r\n")
                while True:
                    gc.collect()
                    data = cl.readline().decode("utf-8").rstrip("\r\n")
                    if len(data) <= 0:
                        print("Client disappeared")
                        do_run = False
                        break

                    command = data.split(" ")[0].upper()
                    payload = data[len(command):].lstrip()

                    path = get_absolute_path(cwd, payload)

                    print("Command={}, Payload={}".format(command, payload))

                    if command == "USER":
                        cl.sendall("230 Logged in.\r\n")
                    elif command == "SYST":
                        cl.sendall("215 UNIX Type: L8\r\n")
                    elif command == "NOOP":
                        cl.sendall("200 OK\r\n")
                    elif command == "FEAT":
                        cl.sendall("211 no-features\r\n")
                    elif command == "PWD":
                        cl.sendall('257 "{}"\r\n'.format(cwd))
                    elif command == "CWD":
                        try:
                            files = uos.listdir(path)
                            cwd = path
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "CDUP":
                        cwd = get_absolute_path(cwd, "..")
                        cl.sendall(msg_250_OK)
                    elif command == "TYPE":
                        # probably should switch between binary and not
                        cl.sendall('200 Transfer mode set\r\n')
                    elif command == "SIZE":
                        try:
                            size = uos.stat(path)[6]
                            cl.sendall('213 {}\r\n'.format(size))
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "QUIT":
                        cl.sendall('221 Bye.\r\n')
                        do_run = False
                        break
                    elif command == "PASV":
                        cl.sendall('227 Entering Passive Mode ({},{},{}).\r\n'.format(
                            addr.replace('.',','), DATA_PORT>>8, DATA_PORT%256))
                        dataclient, data_addr = datasocket.accept()
                        print("FTP Data connection from:", data_addr)
                    elif command == "LIST" or command == "NLST":
                        if not payload.startswith("-"):
                            place = path
                        else:
                            place = cwd
                        try:
                            cl.sendall("150 Here comes the directory listing.\r\n")
                            send_list_data(place, dataclient, command == "LIST" or payload == "-l")
                            cl.sendall("226 Listed.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "RETR":
                        try:
                            cl.sendall("150 Opening data connection.\r\n")
                            send_file_data(path, dataclient)
                            cl.sendall("226 Transfer complete.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "STOR":
                        try:
                            cl.sendall("150 Ok to send data.\r\n")
                            save_file_data(path, dataclient)
                            cl.sendall("226 Transfer complete.\r\n")
                        except:
                            cl.sendall(msg_550_fail)
                        if dataclient is not None:
                            dataclient.close()
                            dataclient = None
                    elif command == "DELE":
                        try:
                            uos.remove(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "RMD":
                        try:
                            uos.rmdir(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "MKD":
                        try:
                            uos.mkdir(path)
                            cl.sendall(msg_250_OK)
                        except:
                            cl.sendall(msg_550_fail)
                    elif command == "RNFR":
                            fromname = path
                            cl.sendall("350 Rename from\r\n")
                    elif command == "RNTO":
                            if fromname is not None:
                                try:
                                    uos.rename(fromname, path)
                                    cl.sendall(msg_250_OK)
                                except:
                                    cl.sendall(msg_550_fail)
                            else:
                                cl.sendall(msg_550_fail)
                            fromname = None
                    elif command == "STAT":
                        if payload == "":
                            cl.sendall("211-Connected to ({})\r\n"
                                       "    Data address ({})\r\n"
                                       "211 TYPE: Binary STRU: File MODE: Stream\r\n".format(
                                       remote_addr[0], addr))
                        else:
                            cl.sendall("213-Directory listing:\r\n")
                            send_list_data(path, cl, True)
                            cl.sendall("213 Done.\r\n")
                    else:
                        cl.sendall("502 Unsupported command.\r\n")
                        print("Unsupported command {} with payload {}".format(command, payload))
            except Exception as err:
                print(err)

            finally:
                cl.close()
                cl = None
    finally:
        datasocket.close()
        ftpsocket.close()
        if dataclient is not None:
            dataclient.close()

ftpserver()
<|endoftext|>


<commit_before>import sys
import os
import time
import shlex
import urllib
import urllib2
from urllib2 import URLError
import re
from functools import partial
if os.name == 'posix' and sys.version_info[0] < 3:
    import subprocess32 as subprocess
else:
    import subprocess
from multiprocessing import Process

import click
from daemonize import Daemonize
from pygments import highlight
from pygments.lexers import PythonLexer, HttpLexer
from pygments.lexers.special import TextLexer
from pygments.formatters import TerminalFormatter

SH_CHECK_DEV = "ps -eo pid,command | grep 'python dev_appserver.py' | grep -v grep | awk '{print $1}'"
SH_KILL_DEV = SH_CHECK_DEV + " | xargs kill"
SH_KILL_SELF = "ps -eo pid,command | grep 'python gae.py' | grep -v grep | grep -v stop | awk '{print $1}' | xargs kill"

def is_dev_server_running():
    out = subprocess.check_output(SH_CHECK_DEV, shell=True)
    if out:
        return True
    else:
        return False

def delay_to_show_server_status():
    time.sleep(2)
    click.echo("")
    subprocess.call("python gae.py status", shell=True)
    click.echo("\nPress <Enter> to continue ...")

def stop_dev_server():
    try:
        subprocess.check_call(SH_KILL_DEV, shell=True)
        subprocess.call(SH_KILL_SELF, shell=True)
        click.secho("[Done]", fg="green")
    except subprocess.CalledProcessError as e:
        click.secho("[Failed]", fg="red")

def load_config_file(config_path):
    try:
        if not config_path:
            config_path = click.get_app_dir("Gae Helper", force_posix=True)
            sys.path.insert(0, config_path)
            import config as cfg
        else:
            sys.path.insert(0, os.path.dirname(config_path))
            fn = os.path.basename(config_path)
            cfg = __import__(fn[:-3], globals(), locals())
        return cfg
    except ImportError as e:
        click.echo("[Error] Can not import Python config file:\n" + str(e))
        return

def get_request_filetype(line):
    regex = r".*module\.py\:\d+\].*\.(.*) HTTP/1\.1"
    matches = re.findall(regex, line)
    if not matches:
        return ""
    return matches[0]

def is_http_request_log(line):
    regex = r".*module\.py\:\d+\].*?\".*?\" \d+.*"
    matches = re.findall(regex, line)
    return True if matches else False

def is_server_status_log(line):
    regex = r".*(admin_server|dispatcher|api_server)\.py\:\d+\] ((?!HTTP/1\.1).)*$"
    matches = re.findall(regex, line)
    return True if matches else False

def is_user_custom_log(line):
    regex = r".*\.py\:\d+\] ((?!HTTP/1\.1).)*$"
    matches = re.findall(regex, line)
    return True if matches else False

def filter_output(line, cfg):
    if hasattr(cfg, "filetype_ignore_filter") and cfg.filetype_ignore_filter:
        ft = get_request_filetype(line)
        if ft in cfg.filetype_ignore_filter:
            return ""
    return line.strip()

def print_server_status_log(line, cfg):
    h_line = highlight_log(line, TextLexer(), cfg)
    click.secho(h_line, nl=False, fg="green")

def print_http_request_log(line, cfg):
    header, body = line.split("]", 1)
    h_header = highlight_log(header, TextLexer(), cfg).strip() + "] "
    h_body = highlight_log(body, HttpLexer(), cfg)
    click.echo(h_header, nl=False)
    click.echo(h_body, nl=False)

def print_user_custom_log(line, cfg):
    header, body = line.split("]", 1)
    h_header = highlight_log(header, TextLexer(), cfg).strip() + "] "
    h_body = highlight_log(body, TextLexer(), cfg)
    click.echo(h_header, nl=False)
    click.secho(h_body, nl=False, bold=True)

def print_python_code(line, cfg):
    h_string = highlight_log(line, PythonLexer(), cfg)
    click.echo(h_string, nl=False)

def highlight_log(string, lexer, cfg):
    if hasattr(cfg, "word_highlight_filter") and cfg.word_highlight_filter:
        lexer.add_filter("highlight", names=cfg.word_highlight_filter)
    return highlight(string, lexer, TerminalFormatter(bg="dark"))

def run_dev_server(cmd, cfg):
    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, universal_newlines=True)
    while True:
        line = ""
        is_pdb = False
        while True:
            ch = process.stdout.read(1)
            if ch != "\n":
                line += ch
            else:
                break
            if line.startswith('(Pdb) '):
                is_pdb = True
                break

        line = filter_output(line, cfg)
        if line:
            if is_pdb:
                click.echo("(Pdb) ", nl=False)
            elif is_server_status_log(line):
                print_server_status_log(line, cfg)
            elif is_http_request_log(line):
                print_http_request_log(line, cfg)
            elif is_user_custom_log(line):
                print_user_custom_log(line, cfg)
            else:
                print_python_code(line, cfg)
        sys.stdout.flush()
        if line == '' and process.poll() is not None:
            break
    return process.poll()

def construct_run_server_cmd(cfg, dev_appserver_options):
    if hasattr(cfg, "gae_sdk_path") and cfg.gae_sdk_path:
        cmd = "cd " + cfg.gae_sdk_path
    else:
        cmd = "cd /usr/local/google_appengine"
    cmd += " && python dev_appserver.py "
    if hasattr(cfg, "project_path") and cfg.project_path:
        cmd += cfg.project_path
    if hasattr(cfg, "datastore_path") and cfg.datastore_path:
        cmd += " --datastore_path=" + cfg.datastore_path
    cmd += " " + " ".join(dev_appserver_options) + " 2>&1"
    return cmd

@click.group()
def gae():
    # before hook
    pass

@gae.command()
def status():
    """Show running status"""
    if is_dev_server_running():
        click.secho("Dev server is running", fg="green")
    else:
        click.secho("Dev server is stopped", fg="red")

@gae.command()
def stop():
    """Stop your local dev server"""
    stop_dev_server()

@gae.command()
@click.option('-p', '--page', 'page', default="",
              help='e.g. --page console')
def admin(page):
    """Launch your GAE admin page in the browser"""
    click.launch('http://localhost:8000/' + page)

@gae.command()
@click.option('-c', '--config', 'config_path', nargs=1, type=click.Path(exists=True),
              help='e.g. --config config.py')
@click.argument('dev_appserver_options', nargs=-1)
def run(config_path, dev_appserver_options):
    """Run your GAE local dev server"""
    if is_dev_server_running():
        click.echo("[Error] Your local dev server is already running")
        return

    cfg = load_config_file(config_path)
    if not cfg:
        return

    cmd = construct_run_server_cmd(cfg, dev_appserver_options)
    run_dev_server(cmd, cfg)

@gae.command()
@click.option('-c', '--config', 'config_path', nargs=1, type=click.Path(exists=True),
              help='e.g. --config config.py')
@click.argument('dev_appserver_options', nargs=-1)
def daemon(config_path, dev_appserver_options):
    """Start your GAE local dev server as a daemon"""
    if is_dev_server_running():
        click.echo("[Error] Your local dev server is already running")
        return

    cfg = load_config_file(config_path)
    if not cfg:
        return

    cmd = construct_run_server_cmd(cfg, dev_appserver_options)
    p = Process(target=delay_to_show_server_status, args=())
    p.start()
    daemon = Daemonize(app="gae_helper", pid="/tmp/gae_dev_helper.pid", action=partial(run_dev_server, cmd, cfg))
    daemon.start()

@gae.command()
@click.option('-c', '--code', 'code', nargs=1, type=click.STRING,
              help='e.g. --code print("Hello World")')
@click.option('-f', '--file', 'f', nargs=1, type=click.File('rb'),
              help='e.g. --file sample.py')
@click.option('-s', '--stream', is_flag=True,
              help='e.g. cat sample.py | python gae.py interactive --stream')
def interactive(code, f, stream):
    """Run your code in interactive console"""
    if not code and not f and not stream:
        click.echo("[Error] Use --code or --file or --stream\n")
        return
    url = "http://localhost:8000/console"
    try:
        response = urllib2.urlopen(url)
    except URLError as e:
        click.echo("[Error] Can not connect to url:\n" + str(e))
        return

    regex = r"'xsrf_token': '(\w*)'"
    matches = re.findall(regex, response.read())
    if not matches:
        click.echo("[Error] Can not find XSRF Token to run your interactive command")
        return

    rpc_code = ""
    if f:
        rpc_code = f.read()
    if code:
        rpc_code += "\n" + code
    if stream:
        rpc_code += "\n" + click.get_text_stream('stdin').read()

    query_args = {'module_name': 'default',
                  'code'       : rpc_code,
                  'xsrf_token' : matches[0]}
    data = urllib.urlencode(query_args)
    request = urllib2.Request(url, data)
    response = urllib2.urlopen(request)
    html = response.read()

    click.echo(html)

if __name__ == '__main__':
    gae()
<commit_msg>add remote_api cmd (support --dev)<commit_after>import sys
import os
import time
import shlex
import urllib
import urllib2
from urllib2 import URLError
import re
from functools import partial
if os.name == 'posix' and sys.version_info[0] < 3:
    import subprocess32 as subprocess
else:
    import subprocess
from multiprocessing import Process

import click
from daemonize import Daemonize
from pygments import highlight
from pygments.lexers import PythonLexer, HttpLexer
from pygments.lexers.special import TextLexer
from pygments.formatters import TerminalFormatter

SH_CHECK_DEV = "ps -eo pid,command | grep 'python dev_appserver.py' | grep -v grep | awk '{print $1}'"
SH_KILL_DEV = SH_CHECK_DEV + " | xargs kill"
SH_KILL_SELF = "ps -eo pid,command | grep 'python gae.py' | grep -v grep | grep -v stop | awk '{print $1}' | xargs kill"

def is_dev_server_running():
    out = subprocess.check_output(SH_CHECK_DEV, shell=True)
    if out:
        return True
    else:
        return False

def delay_to_show_server_status():
    time.sleep(2)
    click.echo("")
    subprocess.call("python gae.py status", shell=True)
    click.echo("\nPress <Enter> to continue ...")

def stop_dev_server():
    try:
        subprocess.check_call(SH_KILL_DEV, shell=True)
        subprocess.call(SH_KILL_SELF, shell=True)
        click.secho("[Done]", fg="green")
    except subprocess.CalledProcessError as e:
        click.secho("[Failed]", fg="red")

def load_config_file(config_path):
    try:
        if not config_path:
            config_path = click.get_app_dir("Gae Helper", force_posix=True)
            sys.path.insert(0, config_path)
            import config as cfg
        else:
            sys.path.insert(0, os.path.dirname(config_path))
            fn = os.path.basename(config_path)
            cfg = __import__(fn[:-3], globals(), locals())
        return cfg
    except ImportError as e:
        click.echo("[Error] Can not import Python config file:\n" + str(e))
        return

def get_request_filetype(line):
    regex = r".*module\.py\:\d+\].*\.(.*) HTTP/1\.1"
    matches = re.findall(regex, line)
    if not matches:
        return ""
    return matches[0]

def is_http_request_log(line):
    regex = r".*module\.py\:\d+\].*?\".*?\" \d+.*"
    matches = re.findall(regex, line)
    return True if matches else False

def is_server_status_log(line):
    regex = r".*(admin_server|dispatcher|api_server)\.py\:\d+\] ((?!HTTP/1\.1).)*$"
    matches = re.findall(regex, line)
    return True if matches else False

def is_user_custom_log(line):
    regex = r".*\.py\:\d+\] ((?!HTTP/1\.1).)*$"
    matches = re.findall(regex, line)
    return True if matches else False

def filter_output(line, cfg):
    if hasattr(cfg, "filetype_ignore_filter") and cfg.filetype_ignore_filter:
        ft = get_request_filetype(line)
        if ft in cfg.filetype_ignore_filter:
            return ""
    return line.strip()

def print_server_status_log(line, cfg):
    h_line = highlight_log(line, TextLexer(), cfg)
    click.secho(h_line, nl=False, fg="green")

def print_http_request_log(line, cfg):
    header, body = line.split("]", 1)
    h_header = highlight_log(header, TextLexer(), cfg).strip() + "] "
    h_body = highlight_log(body, HttpLexer(), cfg)
    click.echo(h_header, nl=False)
    click.echo(h_body, nl=False)

def print_user_custom_log(line, cfg):
    header, body = line.split("]", 1)
    h_header = highlight_log(header, TextLexer(), cfg).strip() + "] "
    h_body = highlight_log(body, TextLexer(), cfg)
    click.echo(h_header, nl=False)
    click.secho(h_body, nl=False, bold=True)

def print_python_code(line, cfg):
    h_string = highlight_log(line, PythonLexer(), cfg)
    click.echo(h_string, nl=False)

def highlight_log(string, lexer, cfg):
    if hasattr(cfg, "word_highlight_filter") and cfg.word_highlight_filter:
        lexer.add_filter("highlight", names=cfg.word_highlight_filter)
    return highlight(string, lexer, TerminalFormatter(bg="dark"))

def run_dev_server(cmd, cfg):
    os.environ['SERVER_SOFTWARE'] = 'Development (devshell remote-api)/1.0'
    os.environ['HTTP_HOST'] = 'localhost'
    process = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True, universal_newlines=True)
    while True:
        line = ""
        is_pdb = False
        while True:
            ch = process.stdout.read(1)
            if ch != "\n":
                line += ch
            else:
                break
            if line.startswith('(Pdb) '):
                is_pdb = True
                break

        line = filter_output(line, cfg)
        if line:
            if is_pdb:
                click.echo("(Pdb) ", nl=False)
            elif is_server_status_log(line):
                print_server_status_log(line, cfg)
            elif is_http_request_log(line):
                print_http_request_log(line, cfg)
            elif is_user_custom_log(line):
                print_user_custom_log(line, cfg)
            else:
                print_python_code(line, cfg)
        sys.stdout.flush()
        if line == '' and process.poll() is not None:
            break
    return process.poll()

def construct_run_server_cmd(cfg, dev_appserver_options):
    if hasattr(cfg, "gae_sdk_path") and cfg.gae_sdk_path:
        cmd = "cd " + cfg.gae_sdk_path
    else:
        cmd = "cd /usr/local/google_appengine"
    cmd += " && python dev_appserver.py "
    if hasattr(cfg, "project_path") and cfg.project_path:
        cmd += cfg.project_path
    if hasattr(cfg, "datastore_path") and cfg.datastore_path:
        cmd += " --datastore_path=" + cfg.datastore_path
    cmd += " " + " ".join(dev_appserver_options) + " 2>&1"
    return cmd

def connect_to_dev_server_by_remote_api(cfg, shell):
    if hasattr(cfg, "gae_sdk_path") and cfg.gae_sdk_path:
        sdk_path = cfg.gae_sdk_path
    else:
        sdk_path = "/usr/local/google_appengine"

    # For transaction
    os.environ['SERVER_SOFTWARE'] = 'Development (devshell remote-api)/1.0'
    # For TaskQueue
    os.environ['HTTP_HOST'] = 'localhost'

    sys.path.append(sdk_path)
    import dev_appserver
    dev_appserver.fix_sys_path()

    if hasattr(cfg, "project_path") and cfg.project_path:
        path = os.path.abspath(os.path.expanduser(cfg.project_path))
        sys.path.insert(0, path)
    if hasattr(cfg, "remote_api_path") and cfg.remote_api_path:
        remote_entry = cfg.remote_api_path
    else:
        remote_entry = "/_ah/remote_api"
    click.echo("use: " + remote_entry)

    from google.appengine.ext.remote_api import remote_api_stub
    remote_api_stub.ConfigureRemoteApiForOAuth("localhost:8080", remote_entry, secure=False)

    try:
        if shell and shell.lower() == "ptpython":
            from ptpython.repl import embed
            embed(globals(), locals())
        else:
            import IPython
            IPython.embed()
    except ImportError:
        click.secho("\nInstall ipython or ptpython to have better life!\n", bold=True, fg="green")
        import code
        code.interact(local=locals())

@click.group()
def gae():
    # before hook
    pass

@gae.command()
def status():
    """Show running status"""
    if is_dev_server_running():
        click.secho("Dev server is running", fg="green")
    else:
        click.secho("Dev server is stopped", fg="red")

@gae.command()
def stop():
    """Stop your local dev server"""
    stop_dev_server()

@gae.command()
@click.option('-p', '--page', 'page', default="",
              help='e.g. --page console')
def admin(page):
    """Launch your GAE admin page in the browser"""
    click.launch('http://localhost:8000/' + page)

@gae.command()
@click.option('-c', '--config', 'config_path', nargs=1, type=click.Path(exists=True),
              help='e.g. --config config.py')
@click.argument('dev_appserver_options', nargs=-1)
def run(config_path, dev_appserver_options):
    """Run your GAE local dev server"""
    if is_dev_server_running():
        click.echo("[Error] Your local dev server is already running")
        return

    cfg = load_config_file(config_path)
    if not cfg:
        return

    cmd = construct_run_server_cmd(cfg, dev_appserver_options)
    run_dev_server(cmd, cfg)

@gae.command()
@click.option('-c', '--config', 'config_path', nargs=1, type=click.Path(exists=True),
              help='e.g. --config config.py')
@click.argument('dev_appserver_options', nargs=-1)
def daemon(config_path, dev_appserver_options):
    """Run your GAE local dev server as a daemon"""
    if is_dev_server_running():
        click.echo("[Error] Your local dev server is already running")
        return

    cfg = load_config_file(config_path)
    if not cfg:
        return

    cmd = construct_run_server_cmd(cfg, dev_appserver_options)
    p = Process(target=delay_to_show_server_status, args=())
    p.start()
    daemon = Daemonize(app="gae_helper", pid="/tmp/gae_dev_helper.pid", action=partial(run_dev_server, cmd, cfg))
    daemon.start()

@gae.command()
@click.option('-c', '--config', 'config_path', nargs=1, type=click.Path(exists=True),
              help='e.g. --config config.py')
@click.option('-d', '--dev', 'dev', is_flag=True,
              help='e.g. --dev')
@click.option('-s', '--shell', 'shell', nargs=1, type=click.STRING, default="ipython",
              help='e.g. --shell ptpython \n# default: ipython')
def remote_api(config_path, dev, shell):
    """Connect to your GAE dev/pro server"""
    if not is_dev_server_running():
        click.echo("[Error] Your local dev server is not running")
        return

    cfg = load_config_file(config_path)
    if not cfg:
        return

    if dev:
        connect_to_dev_server_by_remote_api(cfg, shell)
    else:
        click.echo("[Error] Use --dev or --pro to specify the server you want to connect")

@gae.command()
@click.option('-c', '--code', 'code', nargs=1, type=click.STRING,
              help='e.g. --code print("Hello World")')
@click.option('-f', '--file', 'f', nargs=1, type=click.File('rb'),
              help='e.g. --file sample.py')
@click.option('-s', '--stream', 'stream', is_flag=True,
              help='e.g. cat sample.py | python gae.py interactive --stream')
def interactive(code, f, stream):
    """Run your code in interactive console"""
    if not code and not f and not stream:
        click.echo("[Error] Use --code or --file or --stream\n")
        return
    url = "http://localhost:8000/console"
    try:
        response = urllib2.urlopen(url)
    except URLError as e:
        click.echo("[Error] Can not connect to url:\n" + str(e))
        return

    regex = r"'xsrf_token': '(\w*)'"
    matches = re.findall(regex, response.read())
    if not matches:
        click.echo("[Error] Can not find XSRF Token to run your interactive command")
        return

    rpc_code = ""
    if f:
        rpc_code = f.read()
    if code:
        rpc_code += "\n" + code
    if stream:
        rpc_code += "\n" + click.get_text_stream('stdin').read()

    query_args = {'module_name': 'default',
                  'code'       : rpc_code,
                  'xsrf_token' : matches[0]}
    data = urllib.urlencode(query_args)
    request = urllib2.Request(url, data)
    response = urllib2.urlopen(request)
    html = response.read()

    click.echo(html)

if __name__ == '__main__':
    gae()
<|endoftext|>


<commit_before>import ctypes
from ctypes import Structure, POINTER, c_void_p, c_size_t, c_double, c_char_p, cast

from array import array
import numpy as np

lib = ctypes.cdll.LoadLibrary("libpolyline_ffi.dylib")

class _FFIArray(Structure):
    """
    Convert sequence of float lists to a C-compatible void array
    example: [[1.0, 2.0], [3.0, 4.0]]

    """
    _fields_ = [("data", c_void_p),
                ("len", c_size_t)]

    @classmethod
    def from_param(cls, seq):
        """  Allow implicit conversions """
        return seq if isinstance(seq, cls) else cls(seq)

    def __init__(self, seq, data_type = c_double):
        arr = ((c_double * 2) * len(seq))()
        for i, member in enumerate(seq):
            arr[i][0] = member[0]
            arr[i][1] = member[1]
        self.data = cast(arr, c_void_p)
        self.len = len(seq)

class _CoordResult(Structure):
    """ Container for returned FFI coordinate data """
    _fields_ = [("coords", _FFIArray)]

class _PolylineResult(Structure):
    """ Container for returned FFI Polyline data """
    _fields_ = [("line", c_void_p)]

def _void_array_to_nested_list(res, _func, _args):
    """ Dereference the FFI result to a list of coordinates """
    coords = [list(pair) for pair in
        ((POINTER(c_double * 2).from_buffer_copy(res.coords)[:res.coords.len]))
        ]
    drop_array(res.coords)
    return coords

def void_array_to_string(res, _func, _args):
    """ Dereference the FFI result to a utf8 polyline """
    result = cast(res.line, c_char_p)
    polyline = result.value
    drop_cstring(result)
    return polyline

decode_polyline = lib.decode_polyline_ffi
decode_polyline.argtypes = (c_char_p,)
decode_polyline.restype = _CoordResult
decode_polyline.errcheck = _void_array_to_nested_list

encode_coordinates = lib.encode_coordinates_ffi
encode_coordinates.argtypes = (_FFIArray,)
encode_coordinates.restype = _PolylineResult
encode_coordinates.errcheck = void_array_to_string

# Free FFI-allocated memory
drop_cstring = lib.drop_cstring
drop_cstring.argtypes = (c_char_p,)
drop_cstring.restype = None

drop_array = lib.drop_float_array
drop_array.argtypes = (_FFIArray,)
drop_array.restype = None

pl = "_ibE_seK_seK_seK"
res = decode_polyline(pl)
print res
print encode_coordinates(res)

<commit_msg>Use default datatype<commit_after>import ctypes
from ctypes import Structure, POINTER, c_void_p, c_size_t, c_double, c_char_p, cast

lib = ctypes.cdll.LoadLibrary("libpolyline_ffi.dylib")

class _FFIArray(Structure):
    """
    Convert sequence of float lists to a C-compatible void array
    example: [[1.0, 2.0], [3.0, 4.0]]

    """
    _fields_ = [("data", c_void_p),
                ("len", c_size_t)]

    @classmethod
    def from_param(cls, seq):
        """  Allow implicit conversions """
        return seq if isinstance(seq, cls) else cls(seq)

    def __init__(self, seq, data_type = c_double):
        arr = ((data_type * 2) * len(seq))()
        for i, member in enumerate(seq):
            arr[i][0] = member[0]
            arr[i][1] = member[1]
        self.data = cast(arr, c_void_p)
        self.len = len(seq)

class _CoordResult(Structure):
    """ Container for returned FFI coordinate data """
    _fields_ = [("coords", _FFIArray)]

class _PolylineResult(Structure):
    """ Container for returned FFI Polyline data """
    _fields_ = [("line", c_void_p)]

def _void_array_to_nested_list(res, _func, _args):
    """ Dereference the FFI result to a list of coordinates """
    coords = [list(pair) for pair in
        ((POINTER(c_double * 2).from_buffer_copy(res.coords)[:res.coords.len]))
        ]
    drop_array(res.coords)
    return coords

def void_array_to_string(res, _func, _args):
    """ Dereference the FFI result to a utf8 polyline """
    result = cast(res.line, c_char_p)
    polyline = result.value
    drop_cstring(result)
    return polyline

decode_polyline = lib.decode_polyline_ffi
decode_polyline.argtypes = (c_char_p,)
decode_polyline.restype = _CoordResult
decode_polyline.errcheck = _void_array_to_nested_list

encode_coordinates = lib.encode_coordinates_ffi
encode_coordinates.argtypes = (_FFIArray,)
encode_coordinates.restype = _PolylineResult
encode_coordinates.errcheck = void_array_to_string

# Free FFI-allocated memory
drop_cstring = lib.drop_cstring
drop_cstring.argtypes = (c_char_p,)
drop_cstring.restype = None

drop_array = lib.drop_float_array
drop_array.argtypes = (_FFIArray,)
drop_array.restype = None

pl = "_ibE_seK_seK_seK"
res = decode_polyline(pl)
print(res)
print(encode_coordinates(res))
print("bye")

<|endoftext|>


<commit_before>from __future__ import absolute_import

__all__ = ['dopen', 'dshape', 'cat_dshapes', 'broadcastable',
                'from_ctypes', 'from_cffi']

import operator
import itertools
import ctypes

from . import parser
from .coretypes import DataShape, Fixed, TypeVar, Record, \
                uint8, uint16, uint32, uint64, \
                int8, int16, int32, int64, \
                float32, float64

#------------------------------------------------------------------------
# Utility Functions for DataShapes
#------------------------------------------------------------------------

def dopen(fname):
    contents = open(fname).read()
    return parser.parse_extern(contents)

def dshape(o):
    if isinstance(o, str):
        return parser.parse(o)
    elif isinstance(o, DataShape):
        return o
    else:
        raise TypeError('Cannot create dshape from object of type %s' % type(o))

def cat_dshapes(dslist):
    """
    Concatenates a list of dshapes together along
    the first axis. Raises an error if there is
    a mismatch along another axis or the measures
    are different.

    Requires that the leading dimension be a known
    size for all data shapes.
    TODO: Relax this restriction to support
          streaming dimensions.
    """
    if len(dslist) == 0:
        raise ValueError('Cannot concatenate an empty list of dshapes')
    elif len(dslist) == 1:
        return dslist[0]

    outer_dim_size = operator.index(dslist[0][0])
    inner_ds = dslist[0][1:]
    for ds in dslist[1:]:
        outer_dim_size += operator.index(ds[0])
        if ds[1:] != inner_ds:
            raise ValueError(('The datashapes to concatenate much'
                            ' all match after'
                            ' the first dimension (%s vs %s)') %
                            (inner_ds, ds[1:]))
    return DataShape([Fixed(outer_dim_size)] + list(inner_ds))

def broadcastable(dslist, ranks=None, rankconnect=[]):
    """Return output (outer) shape if datashapes are broadcastable.

    The default is to assume broadcasting over a scalar operation.
    However, if the kernel to be applied takes arrays as arguments,
    then rank and rank-connect provide the inner-shape information with
    ranks a list of integers indicating the kernel rank of each argument
    and rank-connect a list of sets of tuples where each set contains the
    dimensions that must match and each tuple is (argument #, inner-dim #)
    """
    if ranks is None:
        ranks = [0]*len(dslist)

    shapes = [dshape.shape for dshape in dslist]

    # ensure shapes are large enough
    for i, shape, rank in zip(range(len(dslist)), shapes, ranks):
        if len(shape) < rank:
            raise TypeError(('Argument %d is not large-enough '
                            'for kernel rank') % i)

    splitshapes = [(shape[:len(shape)-rank], shape[len(shape)-rank:])
                             for shape, rank in zip(shapes, ranks)]
    outshapes, inshapes = zip(*splitshapes)

    # broadcast outer-dimensions
    maxshape = max(len(shape) for shape in outshapes)
    outshapes = [(1,)*(maxshape-len(shape))+shape for shape in outshapes]

    # check rank-connections
    for shape1, shape2 in itertools.combinations(outshapes, 2):
        if any((dim1 != 1 and dim2 != 1 and dim1 != dim2)
                  for dim1, dim2 in zip(shape1,shape2)):
            raise TypeError('Outer-dimensions are not broadcastable '
                            'to the same shape')
    outshape = tuple(map(max, zip(*outshapes)))

    for connect in rankconnect:
        for (arg1, dim1), (arg2,dim2) in itertools.combinations(connect, 2):
            if (inshapes[arg1][dim1] != inshapes[arg2][dim2]):
                raise TypeError("Inner dimensions do not match in " +
                                "argument %d and argument %d" % (arg1, arg2))

    return outshape

def _from_cffi_internal(ffi, ctype):
    k = ctype.kind
    if k == 'struct':
        # TODO: Assuming the field offsets match
        #       blaze kernels - need to sync up blaze, dynd,
        #       cffi, numpy, etc so that the field offsets always work!
        #       Also need to make sure there are no bitsize/bitshift
        #       values that would be incompatible.
        return Record([(f[0], _from_cffi_internal(ffi, f[1].type))
                        for f in ctype.fields])
    elif k == 'array':
        if ctype.length is None:
            # Only the first array can have the size
            # unspecified, so only need a single name
            dsparams = [TypeVar('N')]
        else:
            dsparams = [Fixed(ctype.length)]
        ctype = ctype.item
        while ctype.kind == 'array':
            dsparams.append(Fixed(ctype.length))
            ctype = ctype.item
        dsparams.append(_from_cffi_internal(ffi, ctype))
        return DataShape(dsparams)
    elif k == 'primitive':
        cn = ctype.cname
        if cn in ['signed char', 'short', 'int',
                        'long', 'long long']:
            so = ffi.sizeof(ctype)
            if so == 1:
                return int8
            elif so == 2:
                return int16
            elif so == 4:
                return int32
            elif so == 8:
                return int64
            else:
                raise TypeError('cffi primitive "%s" has invalid size %d' %
                                (cn, so))
        elif cn in ['unsigned char', 'unsigned short',
                        'unsigned int', 'unsigned long',
                        'unsigned long long']:
            so = ffi.sizeof(ctype)
            if so == 1:
                return uint8
            elif so == 2:
                return uint16
            elif so == 4:
                return uint32
            elif so == 8:
                return uint64
            else:
                raise TypeError('cffi primitive "%s" has invalid size %d' %
                                (cn, so))
        elif cn == 'float':
            return float32
        elif cn == 'double':
            return float64
        else:
            raise TypeError('Unrecognized cffi primitive "%s"' % cn)
    elif k == 'pointer':
        raise TypeError('a pointer can only be at the outer level of a cffi type '
                        'when converting to blaze datashape')
    else:
        raise TypeError('Unrecognized cffi kind "%s"' % k)

def from_cffi(ffi, ctype):
    """
    Constructs a blaze dshape from a cffi type.
    """
    # Allow one pointer dereference at the outermost level
    if ctype.kind == 'pointer':
        ctype = ctype.item
    return _from_cffi_internal(ffi, ctype)

def from_ctypes(ctype):
    """
    Constructs a blaze dshape from a ctypes type.
    """
    if issubclass(ctype, ctypes.Structure):
        fields = []
        for nm, tp in ctype._fields_:
            child_ds = from_ctypes(tp)
            fields.append((nm, from_ctypes(tp)))
        ds = Record(fields)
        # TODO: Validate that the ctypes offsets match
        #       the C offsets blaze uses
        return ds
    elif issubclass(ctype, ctypes.Array):
        dstup = []
        while issubclass(ctype, ctypes.Array):
            dstup.append(Fixed(ctype._length_))
            ctype = ctype._type_
        dstup.append(from_ctypes(ctype))
        return DataShape(tuple(dstup))
    elif ctype == ctypes.c_int8:
        return int8
    elif ctype == ctypes.c_int16:
        return int16
    elif ctype == ctypes.c_int32:
        return int32
    elif ctype == ctypes.c_int64:
        return int64
    elif ctype == ctypes.c_uint8:
        return uint8
    elif ctype == ctypes.c_uint16:
        return uint16
    elif ctype == ctypes.c_uint32:
        return uint32
    elif ctype == ctypes.c_uint64:
        return uint64
    elif ctype == ctypes.c_float:
        return float32
    elif ctype == ctypes.c_double:
        return float64
    else:
        raise TypeError('Cannot convert ctypes %r into '
                        'a blaze datashape')
<commit_msg>Fix datashape.broadcastable function using operator.index and back to Fixed<commit_after>from __future__ import absolute_import

__all__ = ['dopen', 'dshape', 'cat_dshapes', 'broadcastable',
                'from_ctypes', 'from_cffi']

import operator
import itertools
import ctypes

from . import parser
from .coretypes import DataShape, Fixed, TypeVar, Record, \
                uint8, uint16, uint32, uint64, \
                int8, int16, int32, int64, \
                float32, float64

#------------------------------------------------------------------------
# Utility Functions for DataShapes
#------------------------------------------------------------------------

def dopen(fname):
    contents = open(fname).read()
    return parser.parse_extern(contents)

def dshape(o):
    if isinstance(o, str):
        return parser.parse(o)
    elif isinstance(o, DataShape):
        return o
    else:
        raise TypeError('Cannot create dshape from object of type %s' % type(o))

def cat_dshapes(dslist):
    """
    Concatenates a list of dshapes together along
    the first axis. Raises an error if there is
    a mismatch along another axis or the measures
    are different.

    Requires that the leading dimension be a known
    size for all data shapes.
    TODO: Relax this restriction to support
          streaming dimensions.
    """
    if len(dslist) == 0:
        raise ValueError('Cannot concatenate an empty list of dshapes')
    elif len(dslist) == 1:
        return dslist[0]

    outer_dim_size = operator.index(dslist[0][0])
    inner_ds = dslist[0][1:]
    for ds in dslist[1:]:
        outer_dim_size += operator.index(ds[0])
        if ds[1:] != inner_ds:
            raise ValueError(('The datashapes to concatenate much'
                            ' all match after'
                            ' the first dimension (%s vs %s)') %
                            (inner_ds, ds[1:]))
    return DataShape([Fixed(outer_dim_size)] + list(inner_ds))

def broadcastable(dslist, ranks=None, rankconnect=[]):
    """Return output (outer) shape if datashapes are broadcastable.

    The default is to assume broadcasting over a scalar operation.
    However, if the kernel to be applied takes arrays as arguments,
    then rank and rank-connect provide the inner-shape information with
    ranks a list of integers indicating the kernel rank of each argument
    and rank-connect a list of sets of tuples where each set contains the
    dimensions that must match and each tuple is (argument #, inner-dim #)
    """
    if ranks is None:
        ranks = [0]*len(dslist)

    shapes = [tuple(operator.index(s) for s in dshape.shape) for dshape in dslist]

    # ensure shapes are large enough
    for i, shape, rank in zip(range(len(dslist)), shapes, ranks):
        if len(shape) < rank:
            raise TypeError(('Argument %d is not large-enough '
                            'for kernel rank') % i)

    splitshapes = [(shape[:len(shape)-rank], shape[len(shape)-rank:])
                             for shape, rank in zip(shapes, ranks)]
    outshapes, inshapes = zip(*splitshapes)

    # broadcast outer-dimensions
    maxshape = max(len(shape) for shape in outshapes)
    outshapes = [(1,)*(maxshape-len(shape))+shape for shape in outshapes]

    # check rank-connections
    for shape1, shape2 in itertools.combinations(outshapes, 2):
        if any((dim1 != 1 and dim2 != 1 and dim1 != dim2)
                  for dim1, dim2 in zip(shape1,shape2)):
            raise TypeError('Outer-dimensions are not broadcastable '
                            'to the same shape')
    outshape = tuple(map(max, zip(*outshapes)))

    for connect in rankconnect:
        for (arg1, dim1), (arg2,dim2) in itertools.combinations(connect, 2):
            if (inshapes[arg1][dim1] != inshapes[arg2][dim2]):
                raise TypeError("Inner dimensions do not match in " +
                                "argument %d and argument %d" % (arg1, arg2))

    return tuple(Fixed(s) for s in outshape)

def _from_cffi_internal(ffi, ctype):
    k = ctype.kind
    if k == 'struct':
        # TODO: Assuming the field offsets match
        #       blaze kernels - need to sync up blaze, dynd,
        #       cffi, numpy, etc so that the field offsets always work!
        #       Also need to make sure there are no bitsize/bitshift
        #       values that would be incompatible.
        return Record([(f[0], _from_cffi_internal(ffi, f[1].type))
                        for f in ctype.fields])
    elif k == 'array':
        if ctype.length is None:
            # Only the first array can have the size
            # unspecified, so only need a single name
            dsparams = [TypeVar('N')]
        else:
            dsparams = [Fixed(ctype.length)]
        ctype = ctype.item
        while ctype.kind == 'array':
            dsparams.append(Fixed(ctype.length))
            ctype = ctype.item
        dsparams.append(_from_cffi_internal(ffi, ctype))
        return DataShape(dsparams)
    elif k == 'primitive':
        cn = ctype.cname
        if cn in ['signed char', 'short', 'int',
                        'long', 'long long']:
            so = ffi.sizeof(ctype)
            if so == 1:
                return int8
            elif so == 2:
                return int16
            elif so == 4:
                return int32
            elif so == 8:
                return int64
            else:
                raise TypeError('cffi primitive "%s" has invalid size %d' %
                                (cn, so))
        elif cn in ['unsigned char', 'unsigned short',
                        'unsigned int', 'unsigned long',
                        'unsigned long long']:
            so = ffi.sizeof(ctype)
            if so == 1:
                return uint8
            elif so == 2:
                return uint16
            elif so == 4:
                return uint32
            elif so == 8:
                return uint64
            else:
                raise TypeError('cffi primitive "%s" has invalid size %d' %
                                (cn, so))
        elif cn == 'float':
            return float32
        elif cn == 'double':
            return float64
        else:
            raise TypeError('Unrecognized cffi primitive "%s"' % cn)
    elif k == 'pointer':
        raise TypeError('a pointer can only be at the outer level of a cffi type '
                        'when converting to blaze datashape')
    else:
        raise TypeError('Unrecognized cffi kind "%s"' % k)

def from_cffi(ffi, ctype):
    """
    Constructs a blaze dshape from a cffi type.
    """
    # Allow one pointer dereference at the outermost level
    if ctype.kind == 'pointer':
        ctype = ctype.item
    return _from_cffi_internal(ffi, ctype)

def from_ctypes(ctype):
    """
    Constructs a blaze dshape from a ctypes type.
    """
    if issubclass(ctype, ctypes.Structure):
        fields = []
        for nm, tp in ctype._fields_:
            child_ds = from_ctypes(tp)
            fields.append((nm, from_ctypes(tp)))
        ds = Record(fields)
        # TODO: Validate that the ctypes offsets match
        #       the C offsets blaze uses
        return ds
    elif issubclass(ctype, ctypes.Array):
        dstup = []
        while issubclass(ctype, ctypes.Array):
            dstup.append(Fixed(ctype._length_))
            ctype = ctype._type_
        dstup.append(from_ctypes(ctype))
        return DataShape(tuple(dstup))
    elif ctype == ctypes.c_int8:
        return int8
    elif ctype == ctypes.c_int16:
        return int16
    elif ctype == ctypes.c_int32:
        return int32
    elif ctype == ctypes.c_int64:
        return int64
    elif ctype == ctypes.c_uint8:
        return uint8
    elif ctype == ctypes.c_uint16:
        return uint16
    elif ctype == ctypes.c_uint32:
        return uint32
    elif ctype == ctypes.c_uint64:
        return uint64
    elif ctype == ctypes.c_float:
        return float32
    elif ctype == ctypes.c_double:
        return float64
    else:
        raise TypeError('Cannot convert ctypes %r into '
                        'a blaze datashape')
<|endoftext|>