<commit_before>import argparse
import time
import datetime
import json
import os
import re
from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError, Request

def parse_args():
    parser = argparse.ArgumentParser(description="Download all images from a subreddit")
    parser.add_argument('subreddit', help="The subreddit to download")
    parser.add_argument('-s', '--score', help='The minimum number of points a picture must have', type=int, default=0)
    parser.add_argument('-d', '--days', help="The number of days in the past to search through", type=int, default=365)
    parser.add_argument('-o', '--output', help="The directory to download into", type=str, default='downloads')
    parser.add_argument('-t', '--target', help="(Advanced) The target number of posts to download in each interval", type=int, default=20)
    parser.add_argument('-k', '--coefficient', help="(Advanced) The coefficient of change", type=float, default=2.0)
    return parser.parse_args()

def get_secs(t):
    return int(time.mktime(t.timetuple()))

def request(url, *ar, **kwa):
    _retries = kwa.pop('_retries', 4)
    _retry_pause = kwa.pop('_retry_pause', 0)
    res = None
    for _try in xrange(_retries):
        try:
            req = Request(url, headers={"User-Agent" : "reddit-image-extractor by /u/Detry322 - " + ' '.join(os.uname())})
            res = urlopen(req, *ar, **kwa)
        except URLError as exc:
            if _try == _retries - 1:
                return None
            print("Try %r err %r  (%r)" % (
                _try, exc.reason, url))
        else:
            break
    return res

def do_search(subreddit, start, end):
    url = "https://www.reddit.com/r/{}/search.json?q=timestamp:{}..{}&sort=top&restrict_sr=on&syntax=cloudsearch".format(subreddit, get_secs(start), get_secs(end))
    try:
        r = request(url)
        if r is None:
            return []
        data = json.loads(r.read())
        return data["data"]["children"]
    except ValueError:
        return []

def extract_imgur_album_urls(album_url):
    response = request(album_url)
    if not response:
        return []
    info = response.info()

    # Rudimentary check to ensure the URL actually specifies an HTML file
    if 'content-type' in info and not info['content-type'].startswith('text/html'):
        return []

    filedata = response.read()
    # TODO: stop parsing HTML with regexes.
    match = re.compile(r'\"hash\":\"(.[^\"]*)\",\"title\"')
    items = []

    memfile = StringIO.StringIO(filedata)

    for line in memfile.readlines():
        results = re.findall(match, line)
        if not results:
            continue

        items += results

    memfile.close()
    # TODO : url may contain gif image.
    urls = ['http://i.imgur.com/%s.jpg' % (imghash) for imghash in items]

    return urls

def process_imgur_url(url):
    if 'imgur.com/a/' in url or 'imgur.com/gallery/' in url:
        return extract_imgur_album_urls(url)

    # Change .png to .jpg for imgur urls.
    if url.endswith('.png'):
        url = url.replace('.png', '.jpg')
    else:
        # Extract the file extension
        ext = os.path.splitext(os.path.basename(url))[1]
        if ext == '.gifv':
            url = url.replace('.gifv', '.gif')
        if not ext:
            # Append a default
            url += '.jpg'
    return [url]

def extract_urls(url):
    urls = []
    if 'imgur.com' in url:
        urls = process_imgur_url(url)
    else:
        urls = [url]

    return urls

def resize_interval(old_interval, matched_posts, target, coefficient):
    MAX_POSTS = 25
    if matched_posts <= target/4:
        return old_interval*(1+coefficient) # Safety
    difference = float(target - matched_posts)
    if difference > 0:
        return old_interval * ((difference/(3*target/4.0))*coefficient + 1)
    return old_interval * 1.0/(1 + coefficient/3.0*(difference/(target - MAX_POSTS)))

def find_images(subreddit, min_score, days, target, coefficient):
    end = datetime.datetime.now()
    current = end - datetime.timedelta(days=days)
    interval = 3.0
    while current < end:
        current_stop = current + datetime.timedelta(seconds=interval)
        matched_posts = 0
        for post in do_search(subreddit, current, current_stop):
            if post['data']['score'] > min_score:
                matched_posts += 1
                for url in extract_urls(post['data']['url']):
                    yield url
        print " === {} posts matched".format(matched_posts)
        interval = resize_interval(interval, matched_posts, target, coefficient)
        current = current_stop

def download_image(url, download_dir):
    if not os.path.isdir(download_dir):
        os.makedirs(download_dir)

    dest_file = os.path.join(download_dir, os.urandom(10).encode('hex'))

    response = request(url)
    if not response:
        print "Error trying to download file"
        return

    info = response.info()
    actual_url = response.url
    if actual_url == 'http://i.imgur.com/removed.png':
        print "Imgur suggests the image was removed"
        return

    # Work out file type either from the response or the url.
    if 'content-type' in info.keys():
        filetype = info['content-type']
    elif url.endswith('.jpg') or url.endswith('.jpeg'):
        filetype = 'image/jpeg'
    elif url.endswith('.png'):
        filetype = 'image/png'
    elif url.endswith('.gif'):
        filetype = 'image/gif'
    else:
        filetype = 'unknown'

    # Only try to download acceptable image types
    if filetype == 'image/jpeg':
        dest_file += '.jpg'
    elif filetype == 'image/png':
        dest_file += '.png'
    elif filetype == 'image/gif':
        dest_file += '.gif'
    else:
        print 'WRONG FILE TYPE: %s has type: %s!' % (url, filetype)
        return

    filedata = response.read()
    filehandle = open(dest_file, 'wb')
    filehandle.write(filedata)
    filehandle.close()
    print "Downloaded to {}".format(dest_file)

def main():
    args = parse_args()
    for image_url in find_images(args.subreddit, args.score, args.days, args.target, args.coefficient):
        download_image(image_url, args.output)

if __name__ == "__main__":
    main()<commit_msg>Print the interval size<commit_after>import argparse
import time
import datetime
import json
import os
import re
from urllib import urlencode
from urllib2 import urlopen, HTTPError, URLError, Request

def parse_args():
    parser = argparse.ArgumentParser(description="Download all images from a subreddit")
    parser.add_argument('subreddit', help="The subreddit to download")
    parser.add_argument('-s', '--score', help='The minimum number of points a picture must have', type=int, default=0)
    parser.add_argument('-d', '--days', help="The number of days in the past to search through", type=int, default=365)
    parser.add_argument('-o', '--output', help="The directory to download into", type=str, default='downloads')
    parser.add_argument('-t', '--target', help="(Advanced) The target number of posts to download in each interval", type=int, default=20)
    parser.add_argument('-k', '--coefficient', help="(Advanced) The coefficient of change", type=float, default=2.0)
    return parser.parse_args()

def get_secs(t):
    return int(time.mktime(t.timetuple()))

def request(url, *ar, **kwa):
    _retries = kwa.pop('_retries', 4)
    _retry_pause = kwa.pop('_retry_pause', 0)
    res = None
    for _try in xrange(_retries):
        try:
            req = Request(url, headers={"User-Agent" : "reddit-image-extractor by /u/Detry322 - " + ' '.join(os.uname())})
            res = urlopen(req, *ar, **kwa)
        except URLError as exc:
            if _try == _retries - 1:
                return None
            print("Try %r err %r  (%r)" % (
                _try, exc.reason, url))
        else:
            break
    return res

def do_search(subreddit, start, end):
    url = "https://www.reddit.com/r/{}/search.json?q=timestamp:{}..{}&sort=top&restrict_sr=on&syntax=cloudsearch".format(subreddit, get_secs(start), get_secs(end))
    try:
        r = request(url)
        if r is None:
            return []
        data = json.loads(r.read())
        return data["data"]["children"]
    except ValueError:
        return []

def extract_imgur_album_urls(album_url):
    response = request(album_url)
    if not response:
        return []
    info = response.info()

    # Rudimentary check to ensure the URL actually specifies an HTML file
    if 'content-type' in info and not info['content-type'].startswith('text/html'):
        return []

    filedata = response.read()
    # TODO: stop parsing HTML with regexes.
    match = re.compile(r'\"hash\":\"(.[^\"]*)\",\"title\"')
    items = []

    memfile = StringIO.StringIO(filedata)

    for line in memfile.readlines():
        results = re.findall(match, line)
        if not results:
            continue

        items += results

    memfile.close()
    # TODO : url may contain gif image.
    urls = ['http://i.imgur.com/%s.jpg' % (imghash) for imghash in items]

    return urls

def process_imgur_url(url):
    if 'imgur.com/a/' in url or 'imgur.com/gallery/' in url:
        return extract_imgur_album_urls(url)

    # Change .png to .jpg for imgur urls.
    if url.endswith('.png'):
        url = url.replace('.png', '.jpg')
    else:
        # Extract the file extension
        ext = os.path.splitext(os.path.basename(url))[1]
        if ext == '.gifv':
            url = url.replace('.gifv', '.gif')
        if not ext:
            # Append a default
            url += '.jpg'
    return [url]

def extract_urls(url):
    urls = []
    if 'imgur.com' in url:
        urls = process_imgur_url(url)
    else:
        urls = [url]

    return urls

def resize_interval(old_interval, matched_posts, target, coefficient):
    MAX_POSTS = 25
    if matched_posts <= target/4:
        return old_interval*(1+coefficient) # Safety
    difference = float(target - matched_posts)
    if difference > 0:
        return old_interval * ((difference/(3*target/4.0))*coefficient + 1)
    return old_interval * 1.0/(1 + coefficient/3.0*(difference/(target - MAX_POSTS)))

def find_images(subreddit, min_score, days, target, coefficient):
    end = datetime.datetime.now()
    current = end - datetime.timedelta(days=days)
    interval = 60.0
    while current < end:
        print "INTERVAL SIZE: {}".format(interval)
        current_stop = current + datetime.timedelta(seconds=interval)
        matched_posts = 0
        for post in do_search(subreddit, current, current_stop):
            if post['data']['score'] > min_score:
                matched_posts += 1
                for url in extract_urls(post['data']['url']):
                    yield url
        print " === {} posts matched".format(matched_posts)
        interval = resize_interval(interval, matched_posts, target, coefficient)
        current = current_stop

def download_image(url, download_dir):
    if not os.path.isdir(download_dir):
        os.makedirs(download_dir)

    dest_file = os.path.join(download_dir, os.urandom(10).encode('hex'))

    response = request(url)
    if not response:
        print "Error trying to download file"
        return

    info = response.info()
    actual_url = response.url
    if actual_url == 'http://i.imgur.com/removed.png':
        print "Imgur suggests the image was removed"
        return

    # Work out file type either from the response or the url.
    if 'content-type' in info.keys():
        filetype = info['content-type']
    elif url.endswith('.jpg') or url.endswith('.jpeg'):
        filetype = 'image/jpeg'
    elif url.endswith('.png'):
        filetype = 'image/png'
    elif url.endswith('.gif'):
        filetype = 'image/gif'
    else:
        filetype = 'unknown'

    # Only try to download acceptable image types
    if filetype == 'image/jpeg':
        dest_file += '.jpg'
    elif filetype == 'image/png':
        dest_file += '.png'
    elif filetype == 'image/gif':
        dest_file += '.gif'
    else:
        print 'WRONG FILE TYPE: %s has type: %s!' % (url, filetype)
        return

    filedata = response.read()
    filehandle = open(dest_file, 'wb')
    filehandle.write(filedata)
    filehandle.close()
    print "Downloaded to {}".format(dest_file)

def main():
    args = parse_args()
    for image_url in find_images(args.subreddit, args.score, args.days, args.target, args.coefficient):
        download_image(image_url, args.output)

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>import base64
import json
import logging
import os
import re
import threading
from collections import Mapping
from datetime import datetime
from inspect import getfullargspec
from pathlib import Path
from pkgutil import iter_modules
from typing import Any, Optional, Union, Generator, Tuple, Callable

from bs4 import BeautifulSoup
from requests import Response
from torrentool.api import Torrent

if False:  # pragma: nocover
    from .base_tracker import GenericTracker  # noqa
    from .base_rpc import BaseRPC  # noqa
    from .base_bot import BaseBot  # noqa
    from .base_notifier import BaseNotifier  # noqa

LOGGER = logging.getLogger(__name__)

_THREAD_LOCAL = threading.local()

# This regex is used to get hyperlink from torrent comment.
RE_LINK = re.compile(r'(?P<url>https?://[^\s]+)')

def encode_value(value: str, encoding: str = None) -> Union[str, bytes]:
    """Encodes a value.

    :param value:
    :param encoding: Encoding charset.

    """
    if encoding is None:
        return value

    return value.encode(encoding)

def base64encode(string_or_bytes: Union[str, bytes]) -> bytes:
    """Return base64 encoded input

    :param string_or_bytes:

    """
    if isinstance(string_or_bytes, str):
        string_or_bytes = string_or_bytes.encode()

    return base64.encodebytes(string_or_bytes).decode('ascii').encode()

class GlobalParam:
    """Represents global parameter value holder.
    Global params can used anywhere in torrt.

    """
    @staticmethod
    def set(name: str, value: Any):
        setattr(_THREAD_LOCAL, name, value)

    @staticmethod
    def get(name: str) -> Any:
        return getattr(_THREAD_LOCAL, name, None)

def dump_contents(filename: str, contents: Union[bytes, Response]):
    """Dumps contents into a file with a given name.

    :param filename:
    :param contents:

    """
    dump_into = GlobalParam.get('dump_into')

    if not dump_into:
        return

    if hasattr(contents, 'encode_contents'):
        # soup
        text = contents.encode_contents()

    else:
        # requests lib response
        text = contents.content

    with open(str(Path(dump_into) / filename), 'wb') as f:
        f.write(text)

def configure_entity(
        type_name: str,
        registry, alias: str,
        settings_dict: dict = None,
        *,
        before_save: Callable = None
) -> Optional['WithSettings']:
    """Configures and spawns objects using given settings.

    Successful configuration is saved.

    :param type_name: Entity type name to be used in massages.

    :param registry: Registry object.

    :param alias: Entity alias.

    :param settings_dict: Settings dictionary to configure object with.

    :param before_save: Function to trigger right before configuration is saved.
        Should accept entity object as argument.

    """
    LOGGER.info(f'Configuring `{alias}` {type_name.lower()} ...')

    entity_cls = registry.get(alias)

    if entity_cls is not None:

        obj = entity_cls.spawn_with_settings(settings_dict or {})
        configured = obj.test_configuration()

        if configured:
            before_save and before_save(obj)
            obj.save_settings()
            LOGGER.info(f'{type_name} `{alias}` is configured')

            return obj

        else:
            LOGGER.error(f'{type_name} `{alias}` configuration failed. Check your settings')

    else:
        LOGGER.error(f'{type_name} `{alias}` is unknown')

def import_classes():
    """Dynamically imports RPC classes and tracker handlers from their directories."""

    LOGGER.debug('Importing RPC classes ...')
    import_from_path('rpc')

    LOGGER.debug('Importing Tracker classes ...')
    import_from_path('trackers')

    LOGGER.debug('Importing Notifier classes ...')
    import_from_path('notifiers')

    LOGGER.debug('Importing Bot classes ...')
    import_from_path('bots')

def import_from_path(path: str):
    """Dynamically imports modules from package.
    It is an .egg-friendly alternative to os.listdir() walking.

    :param path: path under torrt

    """
    for _, pname, ispkg in iter_modules([str(Path(__file__).parent / path)]):
        __import__(f'torrt.{path}.{pname}')

def parse_torrent(torrent: bytes) -> Torrent:
    """Returns Torrent object from torrent contents.

    :param torrent: Torrent file contents.

    """
    return Torrent.from_string(torrent)

def parse_torrent_file(filepath: str) -> Torrent:
    """Reads a torrent file from filesystem and returns information about it.

    :param filepath:

    """
    with open(filepath, 'rb') as f:
        contents = f.read()

    return parse_torrent(contents)

def make_soup(html: str) -> BeautifulSoup:
    """Returns BeautifulSoup object from a html.

    :param html:

    """
    return BeautifulSoup(html, 'lxml')

def get_url_from_string(string: str) -> str:
    """Returns URL from a string, e.g. torrent comment.

    :param string:

    """
    match = RE_LINK.search(string)

    try:
        match = match.group('url')

    except AttributeError:
        match = ''

    return match

def get_iso_from_timestamp(ts: int) -> str:
    """Get ISO formatted string from timestamp.

    :param ts: timestamp

    """
    return datetime.fromtimestamp(ts).isoformat(' ')

def update_dict(old_dict: dict, new_dict: dict) -> dict:
    """Updates [inplace] old dictionary with data from a new one with respect to existing values.

    :param old_dict:
    :param new_dict:

    """
    for key, val in new_dict.items():

        if isinstance(val, Mapping):
            old_dict[key] = update_dict(old_dict.get(key, {}), val)

        else:
            old_dict[key] = new_dict[key]

    return old_dict

class TorrentData:
    """Represents information about torrent."""

    def __init__(
            self,
            *,
            hash: str = '',
            name: str = '',
            url: str = '',
            url_file: str = '',
            raw: bytes = b'',
            parsed: Torrent = None,
    ):
        self.url = url
        self.url_file = url_file
        self.name = name

        self.raw = raw
        self.parsed = parsed

        self._hash = hash

    def _get_hash(self):
        return self._hash or self.parsed.info_hash

    def _set_hash(self, val: str):
        self._hash = val

    hash = property(_get_hash, _set_hash)

    def to_dict(self) -> dict:
        result = {
            'hash': self.hash,
            'name': self.name,
            'url': self.url,
            'url_file': self.url_file,
        }
        return result

def structure_torrent_data(target_dict: dict, hash_str: str, data: TorrentData):
    """Updated target dict with torrent data structured suitably
    for config storage.

    :param target_dict: dictionary to update inplace
    :param hash_str: torrent identifying hash
    :param data: torrent data (e.g. from tracker page or received from RPC (see parse_torrent()))

    """

    if not data.hash:
        data.hash = hash_str

    target_dict[hash_str] = data.to_dict()

def get_torrent_from_url(url: Optional[str]) -> Optional[TorrentData]:
    """Downloads torrent from a given URL and returns torrent data.

    :param url:

    """
    LOGGER.debug(f'Downloading torrent file from `{url}` ...')

    tracker = TrackerObjectsRegistry.get_for_string(url)  # type: GenericTracker

    if tracker:
        torrent_info = tracker.get_torrent(url)

        if torrent_info is None:
            LOGGER.warning(f'Unable to get torrent from `{url}`')

        else:
            LOGGER.debug(f'Torrent was downloaded from `{url}`')
            return torrent_info

    else:
        LOGGER.warning(f'Tracker handler for `{url}` is not registered')

    return None

def iter_rpc() -> Generator[Tuple[str, 'BaseRPC'], None, None]:
    """Generator to iterate through available and enable RPC objects.
        tuple - rpc_alias, rpc_object

    """
    rpc_objects = RPCObjectsRegistry.get()

    if not rpc_objects:
        LOGGER.error('No RPC objects registered, unable to proceed')
        return

    for rpc_alias, rpc_object in rpc_objects.items():

        if not rpc_object.enabled:
            LOGGER.debug(f'RPC `{rpc_object.alias}` is disabled, skipped.')
            continue

        yield rpc_alias, rpc_object

def iter_bots() -> Generator[Tuple[str, 'BaseBot'], None, None]:
    """Generator to iterate through available bots objects.
        tuple - bot_alias, bot_object

    """
    bot_objects = BotObjectsRegistry.get()

    if not bot_objects:
        LOGGER.error('No Bot objects registered, unable to proceed')
        return

    for alias, object in bot_objects.items():
        yield alias, object

def iter_notifiers() -> Generator[Tuple[str, 'BaseNotifier'], None, None]:
    """Generator to iterate through available notifier objects.
        tuple - notifier_alias, notifier_object

    """
    notifier_objects = NotifierObjectsRegistry.get()

    if not notifier_objects:
        LOGGER.debug('No Notifier registered. Notification skipped')
        return

    for notifier_alias, notifier_object in notifier_objects.items():

        yield notifier_alias, notifier_object

class WithSettings:
    """Introduces settings support for class objects.

    NB: * Settings names are taken from inheriting classes __init__() methods.
        * __init__() method MUST use keyword arguments only.
        * Inheriting classes MUST save settings under object properties with the same name as in __init__().

    """
    alias: str = None

    config_entry_name: str = None
    settings: dict = {}

    def __init__(self, **kwargs):
        pass

    def __str__(self) -> str:
        return self.alias

    @classmethod
    def spawn_with_settings(cls, settings: dict) -> 'WithSettings':
        """Spawns and returns object initialized with given settings.

        :param settings:

        """
        LOGGER.debug(f'Spawning `{cls.__name__}` object with the given settings ...')

        return cls(**settings)

    def save_settings(self):
        """Saves object settings into torrt configuration file."""

        settings = {}

        try:
            settings_names = getfullargspec(self.__init__)[0]

            del settings_names[0]  # do not need `self`

            for name in settings_names:
                settings[name] = getattr(self, name)

        except TypeError:
            pass  # Probably __init__ method is not user-defined.

        config.update({self.config_entry_name: {self.alias: settings}})

class TorrtConfig:
    """Gives methods to work with torrt configuration file."""

    USER_DATA_PATH = Path('~').expanduser() / '.torrt'
    USER_SETTINGS_FILE = USER_DATA_PATH / 'config.json'

    _basic_settings = {
        'time_last_check': 0,
        'walk_interval_hours': 1,
        'rpc': {},
        'trackers': {},
        'torrents': {},
        'notifiers': {},
        'bots': {}
    }

    @classmethod
    def drop_section(cls, realm: str, key: str):
        """Drops config section by its key (name) and updates config.

        :param realm:
        :param key:

        """
        try:
            cfg = cls.load()
            del cfg[realm][key]
            cls.save(cfg)

        except KeyError:
            pass

    @classmethod
    def bootstrap(cls):
        """Initializes configuration file if needed."""

        if not cls.USER_DATA_PATH.exists():
            os.makedirs(str(cls.USER_DATA_PATH))

        if not cls.USER_SETTINGS_FILE.exists():
            cls.save(cls._basic_settings)

        # My precious.
        os.chmod(str(cls.USER_SETTINGS_FILE), 0o600)

    @classmethod
    def update(cls, settings_dict: dict):
        """Updates configuration file with given settings.

        :param settings_dict:

        """
        cls.save(update_dict(cls.load(), settings_dict))

    @classmethod
    def load(cls) -> dict:
        """Returns current settings dictionary."""

        LOGGER.debug(f'Loading configuration file {cls.USER_SETTINGS_FILE} ...')

        cls.bootstrap()

        with open(str(cls.USER_SETTINGS_FILE)) as f:
            settings = json.load(f)

        # Pick up settings entries added in new version
        # and put them into old user config.
        for key, val in cls._basic_settings.items():
            if key not in settings:
                settings[key] = val

        return settings

    @classmethod
    def save(cls, settings_dict: dict):
        """Saves a given dict as torrt configuration.

        :param settings_dict:

        """
        LOGGER.debug(f'Saving configuration file {cls.USER_SETTINGS_FILE} ...')

        with open(str(cls.USER_SETTINGS_FILE), 'w') as f:
            json.dump(settings_dict, f, indent=4)

config = TorrtConfig

class ObjectsRegistry:

    __slots__ = ['_items']

    def __init__(self):
        self._items = {}

    def add(self, obj: Any):
        """Add an object to registry.

        NB: object MUST have `alias` attribute.

        :param obj:

        """
        name = getattr(obj, 'alias')

        LOGGER.debug(f'Registering `{name}` from {obj} ...')

        self._items[name] = obj

    def get(self, obj_alias: str = None) -> Union[dict, Any]:
        """Returns registered objects or a definite object by its alias,
        or registry items if no alias provided.

        :param obj_alias:

        """
        if obj_alias is None:
            return self._items

        return self._items.get(obj_alias)

    def get_for_string(self, string: str) -> Optional[Any]:
        """Returns registered object which can handle a given string.

        :param string:

        """
        for name, obj in self._items.items():
            can_handle_method = getattr(obj, 'can_handle', None)

            if can_handle_method and can_handle_method(string):
                return obj

            elif name in string:
                return self._items[name]

        return None

RPCClassesRegistry = ObjectsRegistry()
RPCObjectsRegistry = ObjectsRegistry()
TrackerClassesRegistry = ObjectsRegistry()
TrackerObjectsRegistry = ObjectsRegistry()
NotifierClassesRegistry = ObjectsRegistry()
NotifierObjectsRegistry = ObjectsRegistry()
BotClassesRegistry = ObjectsRegistry()
BotObjectsRegistry = ObjectsRegistry()
<commit_msg>Calculate name in TorrentData<commit_after>import base64
import json
import logging
import os
import re
import threading
from collections import Mapping
from datetime import datetime
from inspect import getfullargspec
from pathlib import Path
from pkgutil import iter_modules
from typing import Any, Optional, Union, Generator, Tuple, Callable

from bs4 import BeautifulSoup
from requests import Response
from torrentool.api import Torrent

if False:  # pragma: nocover
    from .base_tracker import GenericTracker  # noqa
    from .base_rpc import BaseRPC  # noqa
    from .base_bot import BaseBot  # noqa
    from .base_notifier import BaseNotifier  # noqa

LOGGER = logging.getLogger(__name__)

_THREAD_LOCAL = threading.local()

# This regex is used to get hyperlink from torrent comment.
RE_LINK = re.compile(r'(?P<url>https?://[^\s]+)')

def encode_value(value: str, encoding: str = None) -> Union[str, bytes]:
    """Encodes a value.

    :param value:
    :param encoding: Encoding charset.

    """
    if encoding is None:
        return value

    return value.encode(encoding)

def base64encode(string_or_bytes: Union[str, bytes]) -> bytes:
    """Return base64 encoded input

    :param string_or_bytes:

    """
    if isinstance(string_or_bytes, str):
        string_or_bytes = string_or_bytes.encode()

    return base64.encodebytes(string_or_bytes).decode('ascii').encode()

class GlobalParam:
    """Represents global parameter value holder.
    Global params can used anywhere in torrt.

    """
    @staticmethod
    def set(name: str, value: Any):
        setattr(_THREAD_LOCAL, name, value)

    @staticmethod
    def get(name: str) -> Any:
        return getattr(_THREAD_LOCAL, name, None)

def dump_contents(filename: str, contents: Union[bytes, Response]):
    """Dumps contents into a file with a given name.

    :param filename:
    :param contents:

    """
    dump_into = GlobalParam.get('dump_into')

    if not dump_into:
        return

    if hasattr(contents, 'encode_contents'):
        # soup
        text = contents.encode_contents()

    else:
        # requests lib response
        text = contents.content

    with open(str(Path(dump_into) / filename), 'wb') as f:
        f.write(text)

def configure_entity(
        type_name: str,
        registry, alias: str,
        settings_dict: dict = None,
        *,
        before_save: Callable = None
) -> Optional['WithSettings']:
    """Configures and spawns objects using given settings.

    Successful configuration is saved.

    :param type_name: Entity type name to be used in massages.

    :param registry: Registry object.

    :param alias: Entity alias.

    :param settings_dict: Settings dictionary to configure object with.

    :param before_save: Function to trigger right before configuration is saved.
        Should accept entity object as argument.

    """
    LOGGER.info(f'Configuring `{alias}` {type_name.lower()} ...')

    entity_cls = registry.get(alias)

    if entity_cls is not None:

        obj = entity_cls.spawn_with_settings(settings_dict or {})
        configured = obj.test_configuration()

        if configured:
            before_save and before_save(obj)
            obj.save_settings()
            LOGGER.info(f'{type_name} `{alias}` is configured')

            return obj

        else:
            LOGGER.error(f'{type_name} `{alias}` configuration failed. Check your settings')

    else:
        LOGGER.error(f'{type_name} `{alias}` is unknown')

def import_classes():
    """Dynamically imports RPC classes and tracker handlers from their directories."""

    LOGGER.debug('Importing RPC classes ...')
    import_from_path('rpc')

    LOGGER.debug('Importing Tracker classes ...')
    import_from_path('trackers')

    LOGGER.debug('Importing Notifier classes ...')
    import_from_path('notifiers')

    LOGGER.debug('Importing Bot classes ...')
    import_from_path('bots')

def import_from_path(path: str):
    """Dynamically imports modules from package.
    It is an .egg-friendly alternative to os.listdir() walking.

    :param path: path under torrt

    """
    for _, pname, ispkg in iter_modules([str(Path(__file__).parent / path)]):
        __import__(f'torrt.{path}.{pname}')

def parse_torrent(torrent: bytes) -> Torrent:
    """Returns Torrent object from torrent contents.

    :param torrent: Torrent file contents.

    """
    return Torrent.from_string(torrent)

def parse_torrent_file(filepath: str) -> Torrent:
    """Reads a torrent file from filesystem and returns information about it.

    :param filepath:

    """
    with open(filepath, 'rb') as f:
        contents = f.read()

    return parse_torrent(contents)

def make_soup(html: str) -> BeautifulSoup:
    """Returns BeautifulSoup object from a html.

    :param html:

    """
    return BeautifulSoup(html, 'lxml')

def get_url_from_string(string: str) -> str:
    """Returns URL from a string, e.g. torrent comment.

    :param string:

    """
    match = RE_LINK.search(string)

    try:
        match = match.group('url')

    except AttributeError:
        match = ''

    return match

def get_iso_from_timestamp(ts: int) -> str:
    """Get ISO formatted string from timestamp.

    :param ts: timestamp

    """
    return datetime.fromtimestamp(ts).isoformat(' ')

def update_dict(old_dict: dict, new_dict: dict) -> dict:
    """Updates [inplace] old dictionary with data from a new one with respect to existing values.

    :param old_dict:
    :param new_dict:

    """
    for key, val in new_dict.items():

        if isinstance(val, Mapping):
            old_dict[key] = update_dict(old_dict.get(key, {}), val)

        else:
            old_dict[key] = new_dict[key]

    return old_dict

class TorrentData:
    """Represents information about torrent."""

    def __init__(
            self,
            *,
            hash: str = '',
            name: str = '',
            url: str = '',
            url_file: str = '',
            raw: bytes = b'',
            parsed: Torrent = None,
    ):
        self.url = url
        self.url_file = url_file

        self.raw = raw
        self.parsed = parsed

        self._name = name
        self._hash = hash

    def _get_hash(self):
        return self._hash or self.parsed.info_hash

    def _set_hash(self, val: str):
        self._hash = val

    def _get_name(self):
        return self._name or self.parsed.name

    def _set_name(self, val: str):
        self._name = val

    hash = property(_get_hash, _set_hash)
    name = property(_get_name, _set_name)

    def to_dict(self) -> dict:
        result = {
            'hash': self.hash,
            'name': self.name,
            'url': self.url,
            'url_file': self.url_file,
        }
        return result

def structure_torrent_data(target_dict: dict, hash_str: str, data: TorrentData):
    """Updated target dict with torrent data structured suitably
    for config storage.

    :param target_dict: dictionary to update inplace
    :param hash_str: torrent identifying hash
    :param data: torrent data (e.g. from tracker page or received from RPC (see parse_torrent()))

    """

    if not data.hash:
        data.hash = hash_str

    target_dict[hash_str] = data.to_dict()

def get_torrent_from_url(url: Optional[str]) -> Optional[TorrentData]:
    """Downloads torrent from a given URL and returns torrent data.

    :param url:

    """
    LOGGER.debug(f'Downloading torrent file from `{url}` ...')

    tracker = TrackerObjectsRegistry.get_for_string(url)  # type: GenericTracker

    if tracker:
        torrent_info = tracker.get_torrent(url)

        if torrent_info is None:
            LOGGER.warning(f'Unable to get torrent from `{url}`')

        else:
            LOGGER.debug(f'Torrent was downloaded from `{url}`')
            return torrent_info

    else:
        LOGGER.warning(f'Tracker handler for `{url}` is not registered')

    return None

def iter_rpc() -> Generator[Tuple[str, 'BaseRPC'], None, None]:
    """Generator to iterate through available and enable RPC objects.
        tuple - rpc_alias, rpc_object

    """
    rpc_objects = RPCObjectsRegistry.get()

    if not rpc_objects:
        LOGGER.error('No RPC objects registered, unable to proceed')
        return

    for rpc_alias, rpc_object in rpc_objects.items():

        if not rpc_object.enabled:
            LOGGER.debug(f'RPC `{rpc_object.alias}` is disabled, skipped.')
            continue

        yield rpc_alias, rpc_object

def iter_bots() -> Generator[Tuple[str, 'BaseBot'], None, None]:
    """Generator to iterate through available bots objects.
        tuple - bot_alias, bot_object

    """
    bot_objects = BotObjectsRegistry.get()

    if not bot_objects:
        LOGGER.error('No Bot objects registered, unable to proceed')
        return

    for alias, object in bot_objects.items():
        yield alias, object

def iter_notifiers() -> Generator[Tuple[str, 'BaseNotifier'], None, None]:
    """Generator to iterate through available notifier objects.
        tuple - notifier_alias, notifier_object

    """
    notifier_objects = NotifierObjectsRegistry.get()

    if not notifier_objects:
        LOGGER.debug('No Notifier registered. Notification skipped')
        return

    for notifier_alias, notifier_object in notifier_objects.items():

        yield notifier_alias, notifier_object

class WithSettings:
    """Introduces settings support for class objects.

    NB: * Settings names are taken from inheriting classes __init__() methods.
        * __init__() method MUST use keyword arguments only.
        * Inheriting classes MUST save settings under object properties with the same name as in __init__().

    """
    alias: str = None

    config_entry_name: str = None
    settings: dict = {}

    def __init__(self, **kwargs):
        pass

    def __str__(self) -> str:
        return self.alias

    @classmethod
    def spawn_with_settings(cls, settings: dict) -> 'WithSettings':
        """Spawns and returns object initialized with given settings.

        :param settings:

        """
        LOGGER.debug(f'Spawning `{cls.__name__}` object with the given settings ...')

        return cls(**settings)

    def save_settings(self):
        """Saves object settings into torrt configuration file."""

        settings = {}

        try:
            settings_names = getfullargspec(self.__init__)[0]

            del settings_names[0]  # do not need `self`

            for name in settings_names:
                settings[name] = getattr(self, name)

        except TypeError:
            pass  # Probably __init__ method is not user-defined.

        config.update({self.config_entry_name: {self.alias: settings}})

class TorrtConfig:
    """Gives methods to work with torrt configuration file."""

    USER_DATA_PATH = Path('~').expanduser() / '.torrt'
    USER_SETTINGS_FILE = USER_DATA_PATH / 'config.json'

    _basic_settings = {
        'time_last_check': 0,
        'walk_interval_hours': 1,
        'rpc': {},
        'trackers': {},
        'torrents': {},
        'notifiers': {},
        'bots': {}
    }

    @classmethod
    def drop_section(cls, realm: str, key: str):
        """Drops config section by its key (name) and updates config.

        :param realm:
        :param key:

        """
        try:
            cfg = cls.load()
            del cfg[realm][key]
            cls.save(cfg)

        except KeyError:
            pass

    @classmethod
    def bootstrap(cls):
        """Initializes configuration file if needed."""

        if not cls.USER_DATA_PATH.exists():
            os.makedirs(str(cls.USER_DATA_PATH))

        if not cls.USER_SETTINGS_FILE.exists():
            cls.save(cls._basic_settings)

        # My precious.
        os.chmod(str(cls.USER_SETTINGS_FILE), 0o600)

    @classmethod
    def update(cls, settings_dict: dict):
        """Updates configuration file with given settings.

        :param settings_dict:

        """
        cls.save(update_dict(cls.load(), settings_dict))

    @classmethod
    def load(cls) -> dict:
        """Returns current settings dictionary."""

        LOGGER.debug(f'Loading configuration file {cls.USER_SETTINGS_FILE} ...')

        cls.bootstrap()

        with open(str(cls.USER_SETTINGS_FILE)) as f:
            settings = json.load(f)

        # Pick up settings entries added in new version
        # and put them into old user config.
        for key, val in cls._basic_settings.items():
            if key not in settings:
                settings[key] = val

        return settings

    @classmethod
    def save(cls, settings_dict: dict):
        """Saves a given dict as torrt configuration.

        :param settings_dict:

        """
        LOGGER.debug(f'Saving configuration file {cls.USER_SETTINGS_FILE} ...')

        with open(str(cls.USER_SETTINGS_FILE), 'w') as f:
            json.dump(settings_dict, f, indent=4)

config = TorrtConfig

class ObjectsRegistry:

    __slots__ = ['_items']

    def __init__(self):
        self._items = {}

    def add(self, obj: Any):
        """Add an object to registry.

        NB: object MUST have `alias` attribute.

        :param obj:

        """
        name = getattr(obj, 'alias')

        LOGGER.debug(f'Registering `{name}` from {obj} ...')

        self._items[name] = obj

    def get(self, obj_alias: str = None) -> Union[dict, Any]:
        """Returns registered objects or a definite object by its alias,
        or registry items if no alias provided.

        :param obj_alias:

        """
        if obj_alias is None:
            return self._items

        return self._items.get(obj_alias)

    def get_for_string(self, string: str) -> Optional[Any]:
        """Returns registered object which can handle a given string.

        :param string:

        """
        for name, obj in self._items.items():
            can_handle_method = getattr(obj, 'can_handle', None)

            if can_handle_method and can_handle_method(string):
                return obj

            elif name in string:
                return self._items[name]

        return None

RPCClassesRegistry = ObjectsRegistry()
RPCObjectsRegistry = ObjectsRegistry()
TrackerClassesRegistry = ObjectsRegistry()
TrackerObjectsRegistry = ObjectsRegistry()
NotifierClassesRegistry = ObjectsRegistry()
NotifierObjectsRegistry = ObjectsRegistry()
BotClassesRegistry = ObjectsRegistry()
BotObjectsRegistry = ObjectsRegistry()
<|endoftext|>


<commit_before>import sqlite3
import sys
import PyOpenWorm as P
from PyOpenWorm import Cell

# XXX: Should we specify somewhere whether we have NetworkX or something else?

class Neighbor(P.Property):
    multiple=True
    def __init__(self,**kwargs):
        P.Property.__init__(self,'neighbor',**kwargs)
        self._conns = []

    def get(self,**kwargs):
        """Get a list of neighboring neurons.

           Parameters
           ----------
           See parameters for PyOpenWorm.connection.Connection

           Returns
           -------
           list of Neuron
        """
        if len(self._conns) > 0:
            for c in self._conns:
                yield c.post_cell()
        else:
            c = P.Connection(pre_cell=self.owner,**kwargs)
            for r in c.load():
                yield r.post_cell()

    def set(self, other, **kwargs):
        c = P.Connection(pre_cell=self.owner,post_cell=other,**kwargs)
        self._conns.append(c)
        return c

    def triples(self,**kwargs):
        for c in self._conns:
            for x in c.triples(**kwargs):
                yield x

class Connection(P.Property):
    multiple=True
    def __init__(self,**kwargs):
        P.Property.__init__(self,'connection',**kwargs)
        self._conns = []

    def get(self,pre_post_or_either='pre',**kwargs):
        """Get a list of connections associated with the owning neuron.

           Parameters
           ----------
           type: What kind of junction to look for.
                        0=all, 1=gap junctions only, 2=all chemical synapses
                        3=incoming chemical synapses, 4=outgoing chemical synapses
           Returns
           -------
           list of Connection
        """
        c = []
        if pre_post_or_either == 'pre':
            c.append(P.Connection(pre_cell=self.owner,**kwargs))
        elif pre_post_or_either == 'post':
            c.append(P.Connection(post_cell=self.owner,**kwargs))
        elif pre_post_or_either == 'either':
            c.append(P.Connection(pre_cell=self.owner,**kwargs))
            c.append(P.Connection(post_cell=self.owner,**kwargs))
        for x in c:
            for r in x.load():
                yield r
    def count(self,pre_post_or_either='pre',syntype=None, *args,**kwargs):
        """Get a list of connections associated with the owning neuron.

           Parameters
           ----------
           See parameters for PyOpenWorm.connection.Connection

           Returns
           -------
           int
               The number of connections matching the paramters given
        """
        # XXX: Turn this into a COUNT query
        options = dict()
        options["pre"] = """
                     ?x c:pre_cell ?z .
                     ?z sp:value <%s> .
                     """ % self.owner.identifier()
        options["post"] = """
                      ?x c:post_cell ?z .
                      ?z sp:value <%s> .
                      """ % self.owner.identifier()
        options["either"] = " { %s } UNION { %s } . " % (options['post'], options['pre'])

        if syntype is not None:
            if syntype.lower() == 'gapjunction':
                syntype='gapJunction'
            syntype_pattern = "FILTER( EXISTS { ?x c:syntype ?v . ?v sp:value \"%s\" . }) ." % syntype
        else:
            syntype_pattern = ''

        q = """
        prefix ow: <http://openworm.org/entities/>
        prefix c: <http://openworm.org/entities/Connection/>
        prefix sp: <http://openworm.org/entities/SimpleProperty/>
        prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
        SELECT (COUNT(?x) as ?count) WHERE {
         %s
         %s
        }
        """ % (options[pre_post_or_either], syntype_pattern)

        res = 0
        for x in self.conf['rdf.graph'].query(q):
            res = x['count']
        return int(res)

    def set(self, conn, **kwargs):
        """Add a connection associated with the owner Neuron

           Parameters
           ----------
           conn : PyOpenWorm.connection.Connection
               connection associated with the owner neuron

           Returns
           -------
           A PyOpenWorm.neuron.Connection
        """
        #XXX: Should this create a Connection here instead?
        assert(isinstance(conn, P.Connection))
        self._conns.append(conn)

    def triples(self,**kwargs):
        for c in self._conns:
            for x in c.triples(**kwargs):
                yield x

class Neuron(Cell):
    """
    A neuron.

    Parameters
    ----------
    name : string
        The name of the neuron.

    Attributes
    ----------
    type : DatatypeProperty
        The neuron type (i.e., sensory, interneuron, motor)
    receptor : DatatypeProperty
        The receptor types associated with this neuron
    innexin : DatatypeProperty
        Innexin types associated with this neuron
    neighbor : Property
        Get neurons connected to this neuron
    connection : Property
        Get connections associated with this neuron
    """
    def __init__(self, name=False, **kwargs):
        Cell.__init__(self,name=name,**kwargs)
        # Get neurons connected to this neuron
        Neighbor(owner=self)
        # Get connections from this neuron
        Connection(owner=self)

        Neuron.DatatypeProperty("type",self, multiple=True)
        Neuron.DatatypeProperty("receptor", self, multiple=True)
        Neuron.DatatypeProperty("innexin", self, multiple=True)
        Neuron.DatatypeProperty("neurotransmitter", self, multiple=True)
        ### Aliases ###
        self.get_neighbors = self.neighbor
        self.receptors = self.receptor

    def GJ_degree(self):
        """Get the degree of this neuron for gap junction edges only

        :returns: total number of incoming and outgoing gap junctions
        :rtype: int
        """

        count = 0
        for item in self['nx'].in_edges_iter(self.name(),data=True):
            if 'GapJunction' in item[2]['synapse']:
                count = count + 1
        for item in self['nx'].out_edges_iter(self.name(),data=True):
            if 'GapJunction' in item[2]['synapse']:
                count = count + 1
        return count

    def Syn_degree(self):
        """Get the degree of a this neuron for chemical synapse edges only

        :returns: total number of incoming and outgoing chemical synapses
        :rtype: int
        """
        count = 0
        for item in self['nx'].in_edges_iter(self.name(),data=True):
            if 'Send' in item[2]['synapse']:
                count = count + 1
        for item in self['nx'].out_edges_iter(self.name(),data=True):
            if 'Send' in item[2]['synapse']:
                count = count + 1
        return count

    def _type_semantic(self):
        """Get type of this neuron (motor, interneuron, sensory)

        Use the semantic database as the source

        :returns: the type
        :rtype: str
        """

        qres = self['rdf.graph'].query(
          """SELECT ?objLabel     #we want to get out the labels associated with the objects
           WHERE {
              ?node ?p '"""+self.name()+"""' .   #we are looking first for the node that is the anchor of all information about the specified neuron
              ?node <http://openworm.org/entities/1515> ?object .# having identified that node, here we match an object associated with the node via the 'is a' property (number 1515)
              ?object rdfs:label ?objLabel  #for the object, look up their plain text label.
            }""")

        type = ''
        for r in qres.result:
            type = str(r[0])

        return type

    def _type_networkX(self):
        """Get type of this neuron (motor, interneuron, sensory)

        Use the networkX representation as the source

        :returns: the type
        :rtype: str
        """
        return self['nx'].node[self.name.one()]['ntype']

    def get_incidents(self, type=0):
        """ Get neurons which synapse at this neuron """
        # Directed graph. Getting accessible _from_ this node
        for item in self['nx'].in_edges_iter(self.name(),data=True):
            if 'GapJunction' in item[2]['synapse']:
                yield item[0]

    def _as_neuroml(self):
       """Return this neuron as a NeuroML representation

          :rtype: libNeuroML.Neuron
       """

<commit_msg>Added a neuropeptide property to the Neuron class<commit_after>import sqlite3
import sys
import PyOpenWorm as P
from PyOpenWorm import Cell

# XXX: Should we specify somewhere whether we have NetworkX or something else?

class Neighbor(P.Property):
    multiple=True
    def __init__(self,**kwargs):
        P.Property.__init__(self,'neighbor',**kwargs)
        self._conns = []

    def get(self,**kwargs):
        """Get a list of neighboring neurons.

           Parameters
           ----------
           See parameters for PyOpenWorm.connection.Connection

           Returns
           -------
           list of Neuron
        """
        if len(self._conns) > 0:
            for c in self._conns:
                yield c.post_cell()
        else:
            c = P.Connection(pre_cell=self.owner,**kwargs)
            for r in c.load():
                yield r.post_cell()

    def set(self, other, **kwargs):
        c = P.Connection(pre_cell=self.owner,post_cell=other,**kwargs)
        self._conns.append(c)
        return c

    def triples(self,**kwargs):
        for c in self._conns:
            for x in c.triples(**kwargs):
                yield x

class Connection(P.Property):
    multiple=True
    def __init__(self,**kwargs):
        P.Property.__init__(self,'connection',**kwargs)
        self._conns = []

    def get(self,pre_post_or_either='pre',**kwargs):
        """Get a list of connections associated with the owning neuron.

           Parameters
           ----------
           type: What kind of junction to look for.
                        0=all, 1=gap junctions only, 2=all chemical synapses
                        3=incoming chemical synapses, 4=outgoing chemical synapses
           Returns
           -------
           list of Connection
        """
        c = []
        if pre_post_or_either == 'pre':
            c.append(P.Connection(pre_cell=self.owner,**kwargs))
        elif pre_post_or_either == 'post':
            c.append(P.Connection(post_cell=self.owner,**kwargs))
        elif pre_post_or_either == 'either':
            c.append(P.Connection(pre_cell=self.owner,**kwargs))
            c.append(P.Connection(post_cell=self.owner,**kwargs))
        for x in c:
            for r in x.load():
                yield r
    def count(self,pre_post_or_either='pre',syntype=None, *args,**kwargs):
        """Get a list of connections associated with the owning neuron.

           Parameters
           ----------
           See parameters for PyOpenWorm.connection.Connection

           Returns
           -------
           int
               The number of connections matching the paramters given
        """
        options = dict()
        options["pre"] = """
                     ?x c:pre_cell ?z .
                     ?z sp:value <%s> .
                     """ % self.owner.identifier()
        options["post"] = """
                      ?x c:post_cell ?z .
                      ?z sp:value <%s> .
                      """ % self.owner.identifier()
        options["either"] = " { %s } UNION { %s } . " % (options['post'], options['pre'])

        if syntype is not None:
            if syntype.lower() == 'gapjunction':
                syntype='gapJunction'
            syntype_pattern = "FILTER( EXISTS { ?x c:syntype ?v . ?v sp:value \"%s\" . }) ." % syntype
        else:
            syntype_pattern = ''

        q = """
        prefix ow: <http://openworm.org/entities/>
        prefix c: <http://openworm.org/entities/Connection/>
        prefix sp: <http://openworm.org/entities/SimpleProperty/>
        prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
        SELECT (COUNT(?x) as ?count) WHERE {
         %s
         %s
        }
        """ % (options[pre_post_or_either], syntype_pattern)

        res = 0
        for x in self.conf['rdf.graph'].query(q):
            res = x['count']
        return int(res)

    def set(self, conn, **kwargs):
        """Add a connection associated with the owner Neuron

           Parameters
           ----------
           conn : PyOpenWorm.connection.Connection
               connection associated with the owner neuron

           Returns
           -------
           A PyOpenWorm.neuron.Connection
        """
        #XXX: Should this create a Connection here instead?
        assert(isinstance(conn, P.Connection))
        self._conns.append(conn)

    def triples(self,**kwargs):
        for c in self._conns:
            for x in c.triples(**kwargs):
                yield x

class Neuron(Cell):
    """
    A neuron.

    Parameters
    ----------
    name : string
        The name of the neuron.

    Attributes
    ----------
    type : DatatypeProperty
        The neuron type (i.e., sensory, interneuron, motor)
    receptor : DatatypeProperty
        The receptor types associated with this neuron
    innexin : DatatypeProperty
        Innexin types associated with this neuron
    neurotransmitter : DatatypeProperty
        Neurotransmitters associated with this neuron
    neuropeptide : DatatypeProperty
        Name of the gene corresponding to the neuropeptide produced by this neuron
    neighbor : Property
        Get neurons connected to this neuron
    connection : Property
        Get connections associated with this neuron
    """
    def __init__(self, name=False, **kwargs):
        Cell.__init__(self,name=name,**kwargs)
        # Get neurons connected to this neuron
        #Neighbor(owner=self)
        # Get connections from this neuron
        #Connection(owner=self)

        Neuron.DatatypeProperty("type",self, multiple=True)
        Neuron.DatatypeProperty("receptor", self, multiple=True)
        Neuron.DatatypeProperty("innexin", self, multiple=True)
        Neuron.DatatypeProperty("neurotransmitter", self, multiple=True)
        Neuron.DatatypeProperty("neuropeptide", self, multiple=True)
        ### Aliases ###
        #self.get_neighbors = self.neighbor
        self.receptors = self.receptor

    def GJ_degree(self):
        """Get the degree of this neuron for gap junction edges only

        :returns: total number of incoming and outgoing gap junctions
        :rtype: int
        """

        count = 0
        for item in self['nx'].in_edges_iter(self.name(),data=True):
            if 'GapJunction' in item[2]['synapse']:
                count = count + 1
        for item in self['nx'].out_edges_iter(self.name(),data=True):
            if 'GapJunction' in item[2]['synapse']:
                count = count + 1
        return count

    def Syn_degree(self):
        """Get the degree of a this neuron for chemical synapse edges only

        :returns: total number of incoming and outgoing chemical synapses
        :rtype: int
        """
        count = 0
        for item in self['nx'].in_edges_iter(self.name(),data=True):
            if 'Send' in item[2]['synapse']:
                count = count + 1
        for item in self['nx'].out_edges_iter(self.name(),data=True):
            if 'Send' in item[2]['synapse']:
                count = count + 1
        return count

    def _type_semantic(self):
        """Get type of this neuron (motor, interneuron, sensory)

        Use the semantic database as the source

        :returns: the type
        :rtype: str
        """

        qres = self['rdf.graph'].query(
          """SELECT ?objLabel     #we want to get out the labels associated with the objects
           WHERE {
              ?node ?p '"""+self.name()+"""' .   #we are looking first for the node that is the anchor of all information about the specified neuron
              ?node <http://openworm.org/entities/1515> ?object .# having identified that node, here we match an object associated with the node via the 'is a' property (number 1515)
              ?object rdfs:label ?objLabel  #for the object, look up their plain text label.
            }""")

        type = ''
        for r in qres.result:
            type = str(r[0])

        return type

    def _type_networkX(self):
        """Get type of this neuron (motor, interneuron, sensory)

        Use the networkX representation as the source

        :returns: the type
        :rtype: str
        """
        return self['nx'].node[self.name.one()]['ntype']

    def get_incidents(self, type=0):
        """ Get neurons which synapse at this neuron """
        # Directed graph. Getting accessible _from_ this node
        for item in self['nx'].in_edges_iter(self.name(),data=True):
            if 'GapJunction' in item[2]['synapse']:
                yield item[0]

    def _as_neuroml(self):
       """Return this neuron as a NeuroML representation

          :rtype: libNeuroML.Neuron
       """

<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
import sys
import unitypack
from PIL import ImageOps

SUPPORTED_FORMATS = (
	"AudioClip",
	"Shader",
	"TextAsset",
	"Texture2D",
)

def get_output_path(filename):
	basedir = "out"
	path = os.path.join(basedir, filename)
	dirs = os.path.dirname(path)
	if not os.path.exists(dirs):
		os.makedirs(dirs)
	return path

def write_to_file(filename, contents, mode="w"):
	path = get_output_path(filename)
	with open(path, mode) as f:
		written = f.write(contents)

	print("Written %i bytes to %r" % (written, path))

def handle_asset(asset):
	print(asset)
	for id, obj in asset.objects.items():
		if obj.type not in SUPPORTED_FORMATS:
			print("Skipping %r" % (obj))
			continue

		d = obj.read()

		if obj.type == "AudioClip":
			write_to_file(d.name + ".fsb", d.data, mode="wb")

		elif obj.type == "Shader":
			write_to_file(d.name + ".cg", d.script)

		elif obj.type == "TextAsset":
			write_to_file(d.name + ".txt", d.script)

		elif obj.type == "Texture2D":
			print("Decoding %r" % (d))
			img = ImageOps.flip(d.image)
			path = get_output_path(d.name + ".png")
			img.save(path)

def main():
	files = sys.argv[1:]
	for file in files:
		if file.endswith(".assets"):
			with open(file, "rb") as f:
				asset = unitypack.Asset.from_file(f)
			handle_asset(asset)
			continue

		with open(file, "rb") as f:
			bundle = unitypack.load(f)

		for asset in bundle.assets:
			handle_asset(asset)

if __name__ == "__main__":
	main()
<commit_msg>Decode FSB samples into their original format<commit_after>#!/usr/bin/env python
import os
import sys
import unitypack
from PIL import ImageOps
from fsb5 import FSB5

SUPPORTED_FORMATS = (
	"AudioClip",
	"Shader",
	"TextAsset",
	"Texture2D",
)

def get_output_path(filename):
	basedir = "out"
	path = os.path.join(basedir, filename)
	dirs = os.path.dirname(path)
	if not os.path.exists(dirs):
		os.makedirs(dirs)
	return path

def write_to_file(filename, contents, mode="w"):
	path = get_output_path(filename)
	with open(path, mode) as f:
		written = f.write(contents)

	print("Written %i bytes to %r" % (written, path))

def handle_asset(asset):
	print(asset)
	for id, obj in asset.objects.items():
		if obj.type not in SUPPORTED_FORMATS:
			print("Skipping %r" % (obj))
			continue

		d = obj.read()

		if obj.type == "AudioClip":
			af = FSB5(d.data)
			for i, sample in enumerate(af.samples):
				if i > 0:
					filename = "%s-%i.%s" % (d.name, i, af.get_sample_extension())
				else:
					filename = "%s.%s" % (d.name, af.get_sample_extension())
				write_to_file(filename, af.rebuild_sample(sample), mode="wb")

		elif obj.type == "Shader":
			write_to_file(d.name + ".cg", d.script)

		elif obj.type == "TextAsset":
			write_to_file(d.name + ".txt", d.script)

		elif obj.type == "Texture2D":
			print("Decoding %r" % (d))
			img = ImageOps.flip(d.image)
			path = get_output_path(d.name + ".png")
			img.save(path)

def main():
	files = sys.argv[1:]
	for file in files:
		if file.endswith(".assets"):
			with open(file, "rb") as f:
				asset = unitypack.Asset.from_file(f)
			handle_asset(asset)
			continue

		with open(file, "rb") as f:
			bundle = unitypack.load(f)

		for asset in bundle.assets:
			handle_asset(asset)

if __name__ == "__main__":
	main()
<|endoftext|>


<commit_before>import fnmatch
import os

from . import build_inputs
from .builtins import builtin
from .file_types import *
from .languages import ext2lang
from .path import Path
from .utils import flatten, iterate, listify, objectify, shell_listify

class Compile(build_inputs.Edge):
    def __init__(self, target, builder, file, include, options, extra_deps):
        self.builder = builder
        self.file = file
        self.include = include
        self.options = options
        self.in_shared_library = False
        build_inputs.Edge.__init__(self, target, extra_deps)

class Link(build_inputs.Edge):
    def __init__(self, target, project_name, builder, files, libs, options,
                 extra_deps):
        # This is just for MSBuild. XXX: Remove this?
        self.project_name = project_name

        self.builder = builder
        self.files = files
        self.libs = libs
        self.options = options
        build_inputs.Edge.__init__(self, target, extra_deps)

class Alias(build_inputs.Edge):
    pass

class Command(build_inputs.Edge):
    def __init__(self, target, cmds, extra_deps):
        self.cmds = cmds
        build_inputs.Edge.__init__(self, target, extra_deps)

#####

@builtin
def header(build, env, name):
    return HeaderFile(name, source=Path.srcdir)

@builtin
def header_directory(build, env, directory):
    return HeaderDirectory(directory, source=Path.srcdir)

@builtin
def object_file(build, env, name=None, file=None, include=None, options=None,
                lang=None, extra_deps=None):
    if file is None:
        raise TypeError('"file" argument must not be None')
    if lang is None:
        lang = ext2lang.get( os.path.splitext(file)[1] )
    source_file = objectify(file, SourceFile, source=Path.srcdir, lang=lang)
    includes = [objectify(i, HeaderDirectory, source=Path.srcdir)
                for i in iterate(include)]

    if name is None:
        name = os.path.splitext(file)[0]

    builder = env.compiler(source_file.lang)
    target = builder.output_file(name, lang)
    build.add_edge(Compile( target, builder, source_file, includes,
                            shell_listify(options), extra_deps ))
    return target

@builtin
def object_files(build, env, files, include=None, options=None, lang=None):
    def make_object(f):
        return object_file(build, env, file=f, include=include,
                           options=options, lang=lang)
    return [objectify(i, ObjectFile, make_object) for i in iterate(files)]

def _link(build, env, mode, project_name, name, files, libs=None,
          include=None, compile_options=None, link_options=None, lang=None,
          extra_deps=None):
    objects = object_files(build, env, files, include, compile_options, lang)

    # Flatten the list of libraries and remove any DynamicLibraries (since they
    # can't be linked to at build time).
    libs = [ objectify(i, Library, ExternalLibrary) for i in
             flatten(iterate(libs)) if not isinstance(i, DynamicLibrary) ]

    builder = env.linker((i.lang for i in objects), mode)
    target = builder.output_file(name)
    rule = Link(target, project_name, builder, objects, libs,
                shell_listify(link_options), extra_deps)
    build.add_edge(rule)
    build.fallback_default = target
    return target, rule

@builtin
def executable(build, env, name, *args, **kwargs):
    return _link(build, env, 'executable', name, name, *args, **kwargs)[0]

@builtin
def static_library(build, env, name, *args, **kwargs):
    return _link(build, env, 'static_library', 'lib' + name, name, *args,
                 **kwargs)[0]

@builtin
def shared_library(build, env, name, *args, **kwargs):
    target, rule = _link(build, env, 'shared_library', 'lib' + name, name,
                         *args, **kwargs)
    for i in rule.files:
        i.creator.in_shared_library = True
    return target

@builtin
def alias(build, env, name, deps):
    target = build_inputs.Phony(name)
    build.add_edge(Alias(target, deps))
    return target

@builtin
def command(build, env, name, cmd=None, cmds=None, extra_deps=None):
    if (cmd is None) == (cmds is None):
        raise ValueError('exactly one of "cmd" or "cmds" must be specified')
    elif cmds is None:
        cmds = [cmd]

    target = build_inputs.Phony(name)
    build.add_edge(Command(target, cmds, extra_deps))
    return target

#####

@builtin
def default(build, env, *args):
    build.default_targets.extend(
        i for i in flatten(args) if i.creator
    )

@builtin
def install(build, env, *args):
    for i in flatten(args):
        if isinstance(i, build_inputs.Directory):
            build.install_targets.directories.append(i)
        else:
            default(build, i)
            build.install_targets.files.append(i)

class TestCase(object):
    def __init__(self, test, options):
        self.test = test
        self.options = options

class TestDriver(object):
    def __init__(self, driver, options):
        self.driver = driver
        self.options = options
        self.tests = []

@builtin
def test(build, env, test, options=None, driver=None):
    test = objectify(test, build_inputs.File, source=Path.srcdir)
    build.tests.targets.append(test)
    case = TestCase(test, shell_listify(options))
    (driver or build.tests).tests.append(case)
    return case

@builtin
def test_driver(build, env, driver, options=None, parent=None):
    driver = objectify(driver, Executable, ExternalExecutable)
    result = TestDriver(driver, shell_listify(options))
    (parent or build.tests).tests.append(result)
    return result

@builtin
def test_deps(build, env, *args):
    build.tests.extra_deps.extend(
        i for i in flatten(args) if i.creator
    )

@builtin
def global_options(build, env, options, lang):
    if not lang in build.global_options:
        build.global_options[lang] = []
    build.global_options[lang].extend(shell_listify(options))

def _walk_flat(path):
    names = os.listdir(path)
    dirs, nondirs = [], []
    for name in names:
        if os.path.isdir(os.path.join(path, name)):
            dirs.append(name)
        else:
            nondirs.append(name)
    yield path, dirs, nondirs

@builtin
def find(build_inputs, env, base='.', name='*', type=None, flat=False):
    results = []
    for b in iterate(base):
        generator = _walk_flat(b) if flat else os.walk(b)
        for path, dirs, files in generator:
            if type != 'f':
                results.extend((
                    os.path.join(path, i) for i in fnmatch.filter(dirs, name)
                ))
            if type != 'd':
                results.extend((
                    os.path.join(path, i) for i in fnmatch.filter(files, name)
                ))
    return results
<commit_msg>Provide ObjectFiles for easily accessing already-defined objects<commit_after>import fnmatch
import os

from . import build_inputs
from .builtins import builtin
from .file_types import *
from .languages import ext2lang
from .path import Path
from .utils import flatten, iterate, listify, objectify, shell_listify

class TestCase(object):
    def __init__(self, target, options):
        self.target = target
        self.options = options

class TestDriver(object):
    def __init__(self, target, options):
        self.target = target
        self.options = options
        self.tests = []

class ObjectFiles(list):
    def __getitem__(self, key):
        if isinstance(key, basestring):
            for i in self:
                if i.creator and i.creator.file.path.path == key:
                    return i
            raise ValueError('{} not found'.format(repr(key)))
        else:
            return list.__getitem__(self, key)

#####

class Compile(build_inputs.Edge):
    def __init__(self, target, builder, file, include, options, extra_deps):
        self.builder = builder
        self.file = file
        self.include = include
        self.options = options
        self.in_shared_library = False
        build_inputs.Edge.__init__(self, target, extra_deps)

class Link(build_inputs.Edge):
    def __init__(self, target, project_name, builder, files, libs, options,
                 extra_deps):
        # This is just for MSBuild. XXX: Remove this?
        self.project_name = project_name

        self.builder = builder
        self.files = files
        self.libs = libs
        self.options = options
        build_inputs.Edge.__init__(self, target, extra_deps)

class Alias(build_inputs.Edge):
    pass

class Command(build_inputs.Edge):
    def __init__(self, target, cmds, extra_deps):
        self.cmds = cmds
        build_inputs.Edge.__init__(self, target, extra_deps)

#####

@builtin
def header(build, env, name):
    return HeaderFile(name, source=Path.srcdir)

@builtin
def header_directory(build, env, directory):
    return HeaderDirectory(directory, source=Path.srcdir)

@builtin
def object_file(build, env, name=None, file=None, include=None, options=None,
                lang=None, extra_deps=None):
    if file is None:
        raise TypeError('"file" argument must not be None')
    if lang is None:
        lang = ext2lang.get( os.path.splitext(file)[1] )
    source_file = objectify(file, SourceFile, source=Path.srcdir, lang=lang)
    includes = [objectify(i, HeaderDirectory, source=Path.srcdir)
                for i in iterate(include)]

    if name is None:
        name = os.path.splitext(file)[0]

    builder = env.compiler(source_file.lang)
    target = builder.output_file(name, lang)
    build.add_edge(Compile( target, builder, source_file, includes,
                            shell_listify(options), extra_deps ))
    return target

@builtin
def object_files(build, env, files, include=None, options=None, lang=None):
    def make_object(f):
        return object_file(build, env, file=f, include=include,
                           options=options, lang=lang)
    return ObjectFiles(objectify(i, ObjectFile, make_object)
                       for i in iterate(files))

def _link(build, env, mode, project_name, name, files, libs=None,
          include=None, compile_options=None, link_options=None, lang=None,
          extra_deps=None):
    objects = object_files(build, env, files, include, compile_options, lang)

    # Flatten the list of libraries and remove any DynamicLibraries (since they
    # can't be linked to at build time).
    libs = [ objectify(i, Library, ExternalLibrary) for i in
             flatten(iterate(libs)) if not isinstance(i, DynamicLibrary) ]

    builder = env.linker((i.lang for i in objects), mode)
    target = builder.output_file(name)
    rule = Link(target, project_name, builder, objects, libs,
                shell_listify(link_options), extra_deps)
    build.add_edge(rule)
    build.fallback_default = target
    return target, rule

@builtin
def executable(build, env, name, *args, **kwargs):
    return _link(build, env, 'executable', name, name, *args, **kwargs)[0]

@builtin
def static_library(build, env, name, *args, **kwargs):
    return _link(build, env, 'static_library', 'lib' + name, name, *args,
                 **kwargs)[0]

@builtin
def shared_library(build, env, name, *args, **kwargs):
    target, rule = _link(build, env, 'shared_library', 'lib' + name, name,
                         *args, **kwargs)
    for i in rule.files:
        i.creator.in_shared_library = True
    return target

@builtin
def alias(build, env, name, deps):
    target = build_inputs.Phony(name)
    build.add_edge(Alias(target, deps))
    return target

@builtin
def command(build, env, name, cmd=None, cmds=None, extra_deps=None):
    if (cmd is None) == (cmds is None):
        raise ValueError('exactly one of "cmd" or "cmds" must be specified')
    elif cmds is None:
        cmds = [cmd]

    target = build_inputs.Phony(name)
    build.add_edge(Command(target, cmds, extra_deps))
    return target

#####

@builtin
def default(build, env, *args):
    build.default_targets.extend(
        i for i in flatten(args) if i.creator
    )

@builtin
def install(build, env, *args):
    for i in flatten(args):
        if isinstance(i, build_inputs.Directory):
            build.install_targets.directories.append(i)
        else:
            default(build, i)
            build.install_targets.files.append(i)

@builtin
def test(build, env, test, options=None, driver=None):
    test = objectify(test, build_inputs.File, source=Path.srcdir)
    build.tests.targets.append(test)
    case = TestCase(test, shell_listify(options))
    (driver or build.tests).tests.append(case)
    return case

@builtin
def test_driver(build, env, driver, options=None, parent=None):
    driver = objectify(driver, Executable, ExternalExecutable)
    result = TestDriver(driver, shell_listify(options))
    (parent or build.tests).tests.append(result)
    return result

@builtin
def test_deps(build, env, *args):
    build.tests.extra_deps.extend(
        i for i in flatten(args) if i.creator
    )

@builtin
def global_options(build, env, options, lang):
    if not lang in build.global_options:
        build.global_options[lang] = []
    build.global_options[lang].extend(shell_listify(options))

def _walk_flat(path):
    names = os.listdir(path)
    dirs, nondirs = [], []
    for name in names:
        if os.path.isdir(os.path.join(path, name)):
            dirs.append(name)
        else:
            nondirs.append(name)
    yield path, dirs, nondirs

@builtin
def find(build_inputs, env, base='.', name='*', type=None, flat=False):
    results = []
    for b in iterate(base):
        generator = _walk_flat(b) if flat else os.walk(b)
        for path, dirs, files in generator:
            if type != 'f':
                results.extend((
                    os.path.join(path, i) for i in fnmatch.filter(dirs, name)
                ))
            if type != 'd':
                results.extend((
                    os.path.join(path, i) for i in fnmatch.filter(files, name)
                ))
    return results
<|endoftext|>