<commit_before>import os
from pelican import signals

def add_gallery_post(generator):

    contentpath = generator.settings.get('PATH')
    gallerycontentpath = os.path.join(contentpath,'images/gallery')

    for article in generator.articles:
        if 'gallery' in article.metadata.keys():
            album = article.metadata.get('gallery')
            galleryimages = []

            articlegallerypath=os.path.join(gallerycontentpath, album)

            if(os.path.isdir(articlegallerypath)):
                for i in os.listdir(articlegallerypath):
                    if os.path.isfile(os.path.join(os.path.join(gallerycontentpath, album), i)):
                        galleryimages.append(i)

            article.album=album
            article.galleryimages=galleryimages

def generate_gallery_page(generator):

    contentpath = generator.settings.get('PATH')
    gallerycontentpath = os.path.join(contentpath,'images/gallery')

    for page in generator.pages:
        if page.metadata.get('template') == 'gallery':
            gallery=dict()

            for a in os.listdir(gallerycontentpath):
                if os.path.isdir(os.path.join(gallerycontentpath, a)):

                    for i in os.listdir(os.path.join(gallerycontentpath, a)):
                        if os.path.isfile(os.path.join(os.path.join(gallerycontentpath, a), i)):
                            gallery.setdefault(a, []).append(i)

            page.gallery=gallery

def register():
    signals.article_generator_finalized.connect(add_gallery_post)
    signals.page_generator_finalized.connect(generate_gallery_page)
<commit_msg>added sorting of images<commit_after>import os
from pelican import signals

def add_gallery_post(generator):

    contentpath = generator.settings.get('PATH')
    gallerycontentpath = os.path.join(contentpath,'images/gallery')

    for article in generator.articles:
        if 'gallery' in article.metadata.keys():
            album = article.metadata.get('gallery')
            galleryimages = []

            articlegallerypath=os.path.join(gallerycontentpath, album)

            if(os.path.isdir(articlegallerypath)):
                for i in os.listdir(articlegallerypath):
                    if os.path.isfile(os.path.join(os.path.join(gallerycontentpath, album), i)):
                        galleryimages.append(i)

            article.album = album
            article.galleryimages = sorted(galleryimages)

def generate_gallery_page(generator):

    contentpath = generator.settings.get('PATH')
    gallerycontentpath = os.path.join(contentpath,'images/gallery')

    for page in generator.pages:
        if page.metadata.get('template') == 'gallery':
            gallery = dict()

            for a in os.listdir(gallerycontentpath):
                if os.path.isdir(os.path.join(gallerycontentpath, a)):

                    for i in os.listdir(os.path.join(gallerycontentpath, a)):
                        if os.path.isfile(os.path.join(os.path.join(gallerycontentpath, a), i)):
                            gallery.setdefault(a, []).append(i)
                    gallery[a].sort()

            page.gallery=gallery

def register():
    signals.article_generator_finalized.connect(add_gallery_post)
    signals.page_generator_finalized.connect(generate_gallery_page)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import re
import bs4
import requests
import traceback
import progressbar
import threadpool
import operator
from threading import Thread
from data_center.models import Course, Department
from data_center.const import week_dict, course_dict
from data_center.const import T_YEAR, C_TERM

BASE_URL = 'http://class-qry.acad.ncku.edu.tw/qry/'
DEPT_URL = 'http://class-qry.acad.ncku.edu.tw/qry/qry001.php?dept_no='
SYM_URL = 'http://class-qry.acad.ncku.edu.tw/syllabus/online_display.php'
URL = 'https://www.ccxp.nthu.edu.tw/ccxp/INQUIRE/JH/6/6.2/6.2.3/JH623002.php'  # noqa
cond = 'a'
YS = str(T_YEAR) + '|' + str(C_TERM)

class Course_Struct(object):
    def __init__(self):
        self.dept = None
        self.no = None
        self.serial = None
        self.ctitle = None
        self.clas = None
        self.time = None
        self.note = None
        self.objective = None
        self.prerequisite = None
        self.credit = None
        self.limit = None
        self.room = None
        self.teacher = None
        self.syllabus = None
        self.etitle = None
        self.optional = None

    def __str__(self):
        return str((self.dept, self.no, self.serial, self.time))

def reterieve_html(url):
    try:
        while True:
            r = requests.get(url)
            if r.status_code == 200:
                return r.content
            elif r.status_code == 404:
                return ''

    except:
        print traceback.format_exc()
        return 'Failed to crawl :' + str(url)

def crawl_syllabus(course):
    url = SYM_URL + "?syear=0%s&sem=%s&co_no=%s&class_code=%s" % (T_YEAR, C_TERM, course.serial, course.clas)

    html = reterieve_html(url).replace('
', '
')
    soup = bs4.BeautifulSoup(html, 'html.parser')
    try:
        course.etitle = soup.find('div', { 'id': 'header' }).find_all('span')[1].find_all('br')[1].get_text()
        if course.etitle is None:
            course.etitle = ""
        course_outline = soup.find('div', { 'id': 'container' })
        [x.extract() for x in course_outline.find_all('div', { 'id': 'header' })]
        course.syllabus = str(course_outline.contents[2])
    except:
        print course.dept + '-' + course.no
        print soup
        print "Error On Parsing Syllabus"

def trim_element(element):
    return element.get_text().strip(" \n\r")

def get_dept_list():
    html = reterieve_html(BASE_URL)
    soup = bs4.BeautifulSoup(html, 'html.parser')

    dept_divs = soup.find_all('div', class_='dept')
    dept_a_s = map(lambda x: x.find('a'), dept_divs)
    dept_names = map(lambda x: x.get_text().strip()[2:4], dept_a_s)

    return dept_names

def collect_course_info(tr):
    tds = tr.find_all('td')

    course = Course_Struct()

    course.dept = trim_element(tds[1])
    course.no = trim_element(tds[2])
    course.serial = trim_element(tds[3])
    course.clas = trim_element(tds[5])
    course.ctitle = trim_element(tds[10])
    course.time = trim_element(tds[16])
    course.note = trim_element(tds[18])
    course.objective = ""
    course.prerequisite = trim_element(tds[19])

    course.teacher = trim_element(tds[13])
    course.room = trim_element(tds[17])

    course.credit = trim_element(tds[12])
    course.credit = int(course.credit) if course.credit.isdigit() else 0

    course.optional = (False if trim_element(tds[11]).encode('utf8') == "必修" else True)

    #XXX: limit should be sum up of selected and vacant slot
    course.limit = trim_element(tds[14])
    course.limit = int(course.limit) if course.limit.isdigit() else 0

    # crawl_syllabus(course)

    return course

def archive_courses(courses):
    for course_it in courses:
        try:
            course, create = Course.objects.get_or_create(no=course_it.dept + '-' + course_it.no)

            course.dept = course_it.dept
            course.code = course_it.dept
            course.serial = course_it.serial
            course.clas = course_it.serial

            course.credit = course_it.credit
            course.time = parse_to_nthu(course_it.time)
            course.time_token = get_token(course.time)
            course.limit = course_it.limit
            course.note = course_it.note
            course.objective = course_it.objective
            course.prerequisite = course_it.prerequisite
            # course.ge = course_it.ge
            course.chi_title = course_it.ctitle
            # XXX:should be etitle
            course.eng_title = course_it.ctitle

            course.teacher = course_it.teacher
            course.room = course_it.room
            # course.syllabus = course_it.syllabus
            # XXX: should enable crawl
            course.syllabus = ""
            course.save()

        except Exception as ex:
            print ex
            print course_it

def crawl_dept_courses(dept_code):
    html = reterieve_html(DEPT_URL + dept_code)
    soup = bs4.BeautifulSoup(html, 'html.parser')

    # dept_name =

    class_list = []
    [class_list.append('course_y' + str(i)) for i in range(0, 8)]

    courses_of_yrs = map(lambda x: soup.find_all('tr', class_=x), class_list)

    courses_trs = reduce(operator.add, courses_of_yrs, [])
    courses = map(collect_course_info, courses_trs)

    courses_without_no = filter(lambda x: x.no == '', courses)
    courses = filter(lambda x: x.no != '', courses)
    courses_needs_merge = filter(lambda x: x.serial in reduce(lambda y,z: y + [z.serial], courses, []), courses_without_no)

    for course in courses_needs_merge:
        print 'Merging: ' + course.serial
        target_course = filter(lambda x: (x.serial == course.serial) and (x.clas == course.clas), courses)[0]
        target_course.time += course.time

    for x in courses:
        print x

    archive_courses(courses)

def crawl_course():
    threads = []

    for dept_code in get_dept_list():
        t = Thread(
            target=crawl_dept_courses,
            args=(dept_code,)
        )
        threads.append(t)
        t.start()

    progress = progressbar.ProgressBar()
    for t in progress(threads):
        t.join()

    # print 'Crawling syllabus...'
    # pool = threadpool.ThreadPool(50)
    # reqs = threadpool.makeRequests(
        # syllabus_2_html,
        # [([ACIXSTORE, course], {}) for course in Course.objects.all()]
    # )
    # [pool.putRequest(req) for req in reqs]
    # pool.wait()

    # print 'Total course information: %d' % Course.objects.count()

def crawl_dept_info(dept_code):
    if 'A' in dept_code:
        # GE courses won't be handle here
        return

    html = reterieve_html(DEPT_URL + dept_code)
    soup = bs4.BeautifulSoup(html, 'html.parser')

    # dept_name =

    class_list = []
    [class_list.append('course_y' + str(i)) for i in range(1, 8)]

    courses_of_yrs = map(lambda x: soup.find_all('tr', class_=x), class_list)

    i = 0
    for course_tr in courses_of_yrs:
        i += 1

        if len(course_tr) != 0:
            courses = map(collect_course_info, course_tr)
            courses = filter(lambda x: x.no != '', courses)
            # courses = filter(lambda x: x.optional == True, courses)

            # Get something like ``EE  103BA``V

            clas_map = {'甲':'A', '乙':'B', '丙':'C'}
            for course in courses:
                if course.clas.encode('utf8') == '':
                    course.clas = '甲'.decode('utf8')
                for key, value in clas_map.iteritems():
                    if key in course.clas.encode('utf8'):
                        dept_name = (dept_code + '  ' + str(int(T_YEAR) - i + 1) + 'B' + value)
                        print dept_name

                        department = Department.objects.get_or_create(
                            dept_name=dept_name)[0]
                        try:
                            course_obj = Course.objects.filter(dept__contains=dept_code).get(no__contains=course.no)
                            department.required_course.add(course_obj)
                        except Exception as ex:
                            print ex
                            print dept_code, course.no, 'gg'
                        department.save()

def crawl_dept():
    threads = []

    for dept_code in get_dept_list():
        t = Thread(
            target=crawl_dept_info,
            args=(dept_code,)
        )
        threads.append(t)
        t.start()
        t.join()

    # progress = progressbar.ProgressBar()
    # for t in progress(threads):

    print 'Total department information: %d' % Department.objects.count()

def parse_to_nthu(s):
    week_num_dict = {'1':'M', '2': 'T', '3': 'W', '4': 'R', '5': 'F', '6': 'S', '7': 'U'}
    s = s + '['

    time_map = {'M':[], 'T':[], 'W':[], 'R':[], 'F':[], 'S':[], 'U':[] }

    time_regex = re.compile('.\].+?\[')
    times = time_regex.findall(s)

    for time in times:
        if 90 > ord(time[2]) > 57 :
            if ord(time[2]) == 78:
                beg = 5
            else:
                beg = ord(time[2]) - 54
        else:
            beg = int(time[2])
            if beg >= 5:
                beg += 1

        if 90 > ord(time[-2]) > 57 :
            if ord(time[-2]) == 78:
                end = 5
            else:
                end = ord(time[-2]) - 54
        else:
            end = int(time[-2])
            if end >= 5:
                end += 1

        for i in range(beg, end + 1):
            if i not in time_map[week_num_dict[time[0]]]:
                time_map[week_num_dict[time[0]]].append(i)

    ss = ""
    for key, value in time_map.iteritems():
        value.sort()
        for v in value:
            ss += str(key)
            if v < 5:
                ss += str(v)
            elif v == 5:
                ss += 'N'
            elif 11 > v > 5:
                ss += str(v - 1)
            else:
                ss += chr(int(v) + 54)

    return ss

def get_token(ss):

    try:
        return week_dict[ss[0]] + course_dict[ss[1]] + ss[2:]
    # return s
    except:
        return ''
<commit_msg>Crawl Syllabus<commit_after># -*- coding: utf-8 -*-

import re
import bs4
import requests
import traceback
import progressbar
import threadpool
import operator
from threading import Thread
from data_center.models import Course, Department
from data_center.const import week_dict, course_dict
from data_center.const import T_YEAR, C_TERM

BASE_URL = 'http://class-qry.acad.ncku.edu.tw/qry/'
DEPT_URL = 'http://class-qry.acad.ncku.edu.tw/qry/qry001.php?dept_no='
SYM_URL = 'http://class-qry.acad.ncku.edu.tw/syllabus/online_display.php'
URL = 'https://www.ccxp.nthu.edu.tw/ccxp/INQUIRE/JH/6/6.2/6.2.3/JH623002.php'  # noqa
cond = 'a'
YS = str(T_YEAR) + '|' + str(C_TERM)

class Course_Struct(object):
    def __init__(self):
        self.dept = None
        self.no = None
        self.serial = None
        self.ctitle = None
        self.clas = None
        self.time = None
        self.note = None
        self.objective = None
        self.prerequisite = None
        self.credit = None
        self.limit = None
        self.room = None
        self.teacher = None
        self.syllabus = None
        self.etitle = None
        self.optional = None

    def __str__(self):
        return str((self.dept, self.no, self.serial, self.time))

def reterieve_html(url):
    try:
        while True:
            r = requests.get(url)
            if r.status_code == 200:
                return r.content
            elif r.status_code == 404:
                return ''

    except:
        print traceback.format_exc()
        return 'Failed to crawl :' + str(url)

def crawl_syllabus(course):
    url = SYM_URL + "?syear=0%s&sem=%s&co_no=%s&class_code=%s" % (T_YEAR, C_TERM, course.serial, course.clas)

    html = reterieve_html(url).replace('
', '
')
    soup = bs4.BeautifulSoup(html, 'html.parser')
    try:
        course.etitle = soup.find('div', { 'id': 'header' }).find_all('span')[1].find_all('br')[1].get_text()
        if course.etitle is None:
            course.etitle = ""
        course_outline = soup.find('div', { 'id': 'container' })
        [x.extract() for x in course_outline.find_all('div', { 'id': 'header' })]
        course.syllabus = str(course_outline.contents[2])
    except:
        print course.dept + '-' + course.no
        print soup
        print "Error On Parsing Syllabus"

def trim_element(element):
    return element.get_text().strip(" \n\r")

def get_dept_list():
    html = reterieve_html(BASE_URL)
    soup = bs4.BeautifulSoup(html, 'html.parser')

    dept_divs = soup.find_all('div', class_='dept')
    dept_a_s = map(lambda x: x.find('a'), dept_divs)
    dept_names = map(lambda x: x.get_text().strip()[2:4], dept_a_s)

    return dept_names

def collect_course_info(tr):
    tds = tr.find_all('td')

    course = Course_Struct()

    course.dept = trim_element(tds[1])
    course.no = trim_element(tds[2])
    course.serial = trim_element(tds[3])
    course.clas = trim_element(tds[5])
    course.ctitle = trim_element(tds[10])
    course.time = trim_element(tds[16])
    course.note = trim_element(tds[18])
    course.objective = ""
    course.prerequisite = trim_element(tds[19])

    course.teacher = trim_element(tds[13])
    course.room = trim_element(tds[17])

    course.credit = trim_element(tds[12])
    course.credit = int(course.credit) if course.credit.isdigit() else 0

    course.optional = (False if trim_element(tds[11]).encode('utf8') == "必修" else True)

    #XXX: limit should be sum up of selected and vacant slot
    course.limit = trim_element(tds[14])
    course.limit = int(course.limit) if course.limit.isdigit() else 0

    # crawl_syllabus(course)

    return course

def archive_courses(courses):
    for course_it in courses:
        try:
            course, create = Course.objects.get_or_create(no=course_it.dept + '-' + course_it.no)

            course.dept = course_it.dept
            course.code = course_it.dept
            course.serial = course_it.serial
            course.clas = course_it.serial

            course.credit = course_it.credit
            course.time = parse_to_nthu(course_it.time)
            course.time_token = get_token(course.time)
            course.limit = course_it.limit
            course.note = course_it.note
            course.objective = course_it.objective
            course.prerequisite = course_it.prerequisite
            # course.ge = course_it.ge
            course.chi_title = course_it.ctitle
            course.eng_title = course_it.etitle

            course.teacher = course_it.teacher
            course.room = course_it.room
            course.syllabus = course_it.syllabus
            course.save()

        except Exception as ex:
            print ex
            print course_it

def crawl_dept_courses(dept_code):
    html = reterieve_html(DEPT_URL + dept_code)
    soup = bs4.BeautifulSoup(html, 'html.parser')

    # dept_name =

    class_list = []
    [class_list.append('course_y' + str(i)) for i in range(0, 8)]

    courses_of_yrs = map(lambda x: soup.find_all('tr', class_=x), class_list)

    courses_trs = reduce(operator.add, courses_of_yrs, [])
    courses = map(collect_course_info, courses_trs)

    courses_without_no = filter(lambda x: x.no == '', courses)
    courses = filter(lambda x: x.no != '', courses)
    courses_needs_merge = filter(lambda x: x.serial in reduce(lambda y,z: y + [z.serial], courses, []), courses_without_no)

    for course in courses_needs_merge:
        print 'Merging: ' + course.serial
        target_course = filter(lambda x: (x.serial == course.serial) and (x.clas == course.clas), courses)[0]
        target_course.time += course.time

    for x in courses:
        print x

    archive_courses(courses)

def crawl_course():
    threads = []

    for dept_code in get_dept_list():
        t = Thread(
            target=crawl_dept_courses,
            args=(dept_code,)
        )
        threads.append(t)
        t.start()

    progress = progressbar.ProgressBar()
    for t in progress(threads):
        t.join()

    # print 'Crawling syllabus...'
    # pool = threadpool.ThreadPool(50)
    # reqs = threadpool.makeRequests(
        # syllabus_2_html,
        # [([ACIXSTORE, course], {}) for course in Course.objects.all()]
    # )
    # [pool.putRequest(req) for req in reqs]
    # pool.wait()

    # print 'Total course information: %d' % Course.objects.count()

def crawl_dept_info(dept_code):
    if 'A' in dept_code:
        # GE courses won't be handle here
        return

    html = reterieve_html(DEPT_URL + dept_code)
    soup = bs4.BeautifulSoup(html, 'html.parser')

    # dept_name =

    class_list = []
    [class_list.append('course_y' + str(i)) for i in range(1, 8)]

    courses_of_yrs = map(lambda x: soup.find_all('tr', class_=x), class_list)

    i = 0
    for course_tr in courses_of_yrs:
        i += 1

        if len(course_tr) != 0:
            courses = map(collect_course_info, course_tr)
            courses = filter(lambda x: x.no != '', courses)
            # courses = filter(lambda x: x.optional == True, courses)

            # Get something like ``EE  103BA``V

            clas_map = {'甲':'A', '乙':'B', '丙':'C'}
            for course in courses:
                if course.clas.encode('utf8') == '':
                    course.clas = '甲'.decode('utf8')
                for key, value in clas_map.iteritems():
                    if key in course.clas.encode('utf8'):
                        dept_name = (dept_code + '  ' + str(int(T_YEAR) - i + 1) + 'B' + value)
                        print dept_name

                        department = Department.objects.get_or_create(
                            dept_name=dept_name)[0]
                        try:
                            course_obj = Course.objects.filter(dept__contains=dept_code).get(no__contains=course.no)
                            department.required_course.add(course_obj)
                        except Exception as ex:
                            print ex
                            print dept_code, course.no, 'gg'
                        department.save()

def crawl_dept():
    threads = []

    for dept_code in get_dept_list():
        t = Thread(
            target=crawl_dept_info,
            args=(dept_code,)
        )
        threads.append(t)
        t.start()
        t.join()

    # progress = progressbar.ProgressBar()
    # for t in progress(threads):

    print 'Total department information: %d' % Department.objects.count()

def parse_to_nthu(s):
    week_num_dict = {'1':'M', '2': 'T', '3': 'W', '4': 'R', '5': 'F', '6': 'S', '7': 'U'}
    s = s + '['

    time_map = {'M':[], 'T':[], 'W':[], 'R':[], 'F':[], 'S':[], 'U':[] }

    time_regex = re.compile('.\].+?\[')
    times = time_regex.findall(s)

    for time in times:
        if 90 > ord(time[2]) > 57 :
            if ord(time[2]) == 78:
                beg = 5
            else:
                beg = ord(time[2]) - 54
        else:
            beg = int(time[2])
            if beg >= 5:
                beg += 1

        if 90 > ord(time[-2]) > 57 :
            if ord(time[-2]) == 78:
                end = 5
            else:
                end = ord(time[-2]) - 54
        else:
            end = int(time[-2])
            if end >= 5:
                end += 1

        for i in range(beg, end + 1):
            if i not in time_map[week_num_dict[time[0]]]:
                time_map[week_num_dict[time[0]]].append(i)

    ss = ""
    for key, value in time_map.iteritems():
        value.sort()
        for v in value:
            ss += str(key)
            if v < 5:
                ss += str(v)
            elif v == 5:
                ss += 'N'
            elif 11 > v > 5:
                ss += str(v - 1)
            else:
                ss += chr(int(v) + 54)

    return ss

def get_token(ss):

    try:
        return week_dict[ss[0]] + course_dict[ss[1]] + ss[2:]
    # return s
    except:
        return ''
<|endoftext|>


<commit_before>#!/bin/python

import logging
import logging.handlers

def log(log_name):
	# Set up a specific logger with our desired output level
	logger = logging.getLogger(log_name)
	handler = logging.FileHandler(log_name+'.log')
	logger.setLevel(logging.DEBUG)
	ch = logging.StreamHandler()
	ch.setLevel(logging.DEBUG)
	# create formatter
	formatter = logging.Formatter("%(asctime)s [%(levelname)s] [%(name)s] %(filename)s line:%(lineno)d : %(message)s")
	# add formatter to ch
	handler.setFormatter(formatter)
	ch.setFormatter(formatter)
	# add ch to logger
	logger.addHandler(ch)
	logger.addHandler(handler)
	return logger
<commit_msg>color logging<commit_after><|endoftext|>


<commit_before>import requests
import urllib2
import robotparser
from bs4 import BeautifulSoup
from urllib2 import URLError
from urllib2 import HTTPError
from urlparse import urlparse

url_seed = "https://www.python.org/"

def robot_txt(url):
    robot = robotparser.RobotFileParser()
    robot.set_url(url)
    robot.read()

    is_allowed = robot.can_fetch("*", url)

    return is_allowed

def is_allowed_url(domain,url):
    if domain in url:
        return True

    return False

def is_path_absolute(url):
    return bool(urlparse(url).netloc)

def get_links(html):
    try:
        soup = BeautifulSoup(html)

        #obtains all attributtes 'href'
        for links in soup.find_all('a', href=True):
            if not "#" in links['href']:
                link = links['href']

                domain_parse = urlparse(url_seed)
                domain = domain_parse.netloc

                if not is_path_absolute(links['href']):
                    link = domain + links['href']

                is_allowed_domain = is_allowed_url(domain,link)

                if is_allowed_domain:
                    print link
    except (HTTPError,URLError) as e:
        print e
        pass

def get_html_page(url):
    req = urllib2.Request(url)
    response = urllib2.urlopen(req)
    html = response.read()

    return html

def main():
    is_allowed = robot_txt(url_seed)

    if is_allowed:
        html_page = get_html_page(url_seed)
        get_links(html_page)

if __name__ == "__main__":
    main()
<commit_msg>Adds verification of duplicate items in the 'get_links' function.<commit_after>import requests
import urllib2
import robotparser
from bs4 import BeautifulSoup
from urllib2 import URLError
from urllib2 import HTTPError
from urlparse import urlparse

url_seed = "https://www.python.org/"

def robot_txt(url):
    robot = robotparser.RobotFileParser()
    robot.set_url(url)
    robot.read()

    is_allowed = robot.can_fetch("*", url)

    return is_allowed

def is_allowed_url(domain,url):
    if domain in url:
        return True

    return False

def is_path_absolute(url):
    return bool(urlparse(url).netloc)

def get_links(html):
    try:
        soup = BeautifulSoup(html)
        link_list = []

        #obtains all attributtes 'href'
        for links in soup.find_all('a', href=True):
            if not "#" in links['href']:
                link = links['href']

                domain_parse = urlparse(url_seed)
                domain = domain_parse.netloc

                if not is_path_absolute(links['href']):
                    link = domain + links['href']

                is_allowed_domain = is_allowed_url(domain,link)

                if is_allowed_domain:
                    if not link in link_list:
                        link_list.append(link)

        return link_list
    except (HTTPError,URLError) as e:
        print e
        pass

def get_html_page(url):
    req = urllib2.Request(url)
    response = urllib2.urlopen(req)
    html = response.read()

    return html

def main():
    is_allowed = robot_txt(url_seed)

    if is_allowed:
        teste =[]
        html_page = get_html_page(url_seed)
        teste = get_links(html_page)

        for item in teste:
            print item

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
The MIT License (MIT)

Copyright (c) 2017 SML

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import itertools
import os
from random import choice

import discord
from __main__ import send_cmd_help
from discord.ext import commands

from cogs.utils.dataIO import dataIO

PATH = os.path.join("data", "crdatae")
CLASH_ROYALE_JSON = os.path.join(PATH, "clashroyale.json")

def grouper(n, iterable, fillvalue=None):
    """Group lists into lists of items.

    grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"""
    args = [iter(iterable)] * n
    return itertools.zip_longest(*args, fillvalue=fillvalue)

def random_discord_color():
    """Return random color as an integer."""
    color = ''.join([choice('0123456789ABCDEF') for x in range(6)])
    color = int(color, 16)
    return discord.Color(value=color)

class BotEmoji:
    """Emojis available in bot."""
    def __init__(self, bot):
        self.bot = bot

    def name(self, name):
        """Emoji by name."""
        for server in self.bot.servers:
            for emoji in server.emojis:
                if emoji.name == name:
                    return '<:{}:{}>'.format(emoji.name, emoji.id)
        return ''

    def key(self, key):
        """Chest emojis by api key name or key.

        name is used by this cog.
        key is values returned by the api.
        Use key only if name is not set
        """
        if key in self.map:
            name = self.map[key]
            return self.name(name)
        return ''

class ClashRoyale:
    """Clash Royale Data."""
    instance = None

    class __ClashRoyale:
        """Singleton."""
        def __init__(self, *args, **kwargs):
            """Init."""
            self.data = dataIO.load_json(CLASH_ROYALE_JSON)

    def __init__(self, *args, **kwargs):
        """Init."""
        if not ClashRoyale.instance:
            ClashRoyale.instance = ClashRoyale.__ClashRoyale(*args, **kwargs)
        else:
            pass

    def __getattr__(self, name):
        return getattr(self.instance, name)

    def card_elixir(self, card):
        """"Elixir of a card."""
        try:
            return self.data["Cards"][card]["elixir"]
        except KeyError:
            return 0

class Card():
    """Clash Royale Card."""

    def __init__(self, key=None, level=None):
        """Init.

        Params
        + name (str). Key in the ClashRoyale.json
        """
        self.key = key
        self.level = level

    @property
    def elixir(self):
        """Elixir value."""
        return ClashRoyale().card_elixir(self.key)

    def emoji(self, be: BotEmoji):
        """Emoji representation of the card."""
        if self.key is None:
            return ''
        name = self.key.replace('-', '')
        return be.name(name)

class Deck():
    """Clash Royale Deck.

    Contains 8 cards.
    """

    def __init__(self, card_keys=None, card_levels=None, rank=0, usage=0):
        """Init.

        Params
        + rank (int). Rank on the leaderboard.
        + cards []. List of card ids (keys in ClashRoyale.json).
        + card_levels []. List of card levels.
        """
        self.rank = rank
        self.usage = usage
        self.cards = [Card(key=key) for key in card_keys]
        if card_levels is not None:
            kl_zip = zip(card_keys, card_levels)
            self.cards = [Card(key=k, level=l) for k, l in kl_zip]

    @property
    def avg_elixir(self):
        """Average elixir of the deck."""
        elixirs = [c.elixir for c in self.cards if c.elixir != 0]
        return sum(elixirs) / len(elixirs)

    @property
    def avg_elixir_str(self):
        """Average elixir with format."""
        return 'Average Elixir: {:.3}'.format(self.avg_elixir)

    def emoji_repr(self, be: BotEmoji, show_levels=False):
        """Emoji representaion."""
        out = []
        for card in self.cards:
            emoji = card.emoji(be)
            level = card.level
            level_str = ''
            if show_levels and level is not None:
                level_str = '`{:.<2}`'.format(level)
            out.append('{}{}'.format(emoji, level_str))
        return ''.join(out)

    def __repr__(self):
        return ' '.join([c.key for c in self.cards])

class CRDataEnhanced:
    """Clash Royale Data - Enchanced options.

    Requires CRData cog to function.
    """

    error_msg = {
        "requires_crdata": (
            "The CRData cog is not installed or loaded. "
            "This cog cannot function without it."
        )
    }

    def __init__(self, bot):
        """Init."""
        self.bot = bot
        self.be = BotEmoji(bot)
        self.clashroyale = ClashRoyale().data

    @commands.group(pass_context=True, no_pm=True)
    async def crdatae(self, ctx):
        """Clash Royale Real-Time Global 200 Leaderboard."""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @crdatae.command(name="leaderboard", aliases=['lb'], pass_context=True, no_pm=True)
    async def crdatae_leaderboard(self, ctx):
        """Leaderboard."""
        crdata = self.bot.get_cog('CRData')
        if crdata is None:
            await self.bot.say(self.error_msg["requires_crdata"])
            return

        data = crdata.get_last_data()
        # decks = data["decks"]

        decks = []
        for rank, deck in enumerate(data["decks"], 1):
            cards = [crdata.sfid_to_id(card["key"]) for card in deck]
            levels = [card["level"] for card in deck]
            decks.append(Deck(card_keys=cards, card_levels=levels, rank=rank))

        # embeds
        per_page = 25
        decks_group = list(grouper(per_page, decks))
        color = random_discord_color()

        for em_id, decks in enumerate(decks_group):

            em = self.embed_decks_leaderboard(
                decks,
                page=(em_id + 1),
                title="Clash Royale: Global Top 200 Decks",
                color=color,
                footer_text="Data provided by http://starfi.re"
            )

            await self.bot.say(embed=em)

            if em_id < len(decks_group) - 1:
                show_next = await self.show_next_page(ctx)
                if not show_next:
                    await self.bot.say("Search results aborted.")
                    break

    def embed_decks_leaderboard(self, decks, **kwargs):
        """Show embed decks.

        Params:
        + page. Current page.
        + per_page. Number of results per page.
        + All parameters supported by Discord Embeds.
        """
        em = discord.Embed(**kwargs)

        page = kwargs.get('page', 1)
        per_page = kwargs.get('per_page', 25)
        show_usage = kwargs.get('show_usage', False)
        footer_text = kwargs.get('footer_text', '')

        for deck_id, deck in enumerate(decks):
            if deck is not None:
                usage_str = ''
                if deck.usage and show_usage:
                    usage_str = '(Usage: {})'.format(deck.usage)
                field_name = "Rank {} {}".format(deck.rank, usage_str)
                field_value = '{}\n{}'.format(
                    deck.emoji_repr(self.be, show_levels=True),
                    deck.avg_elixir_str)
                em.add_field(name=field_name, value=field_value)

        em.set_footer(text=footer_text)
        return em

    @crdatae.command(name="search", pass_context=True, no_pm=True)
    async def crdatae_search(self, ctx, *cards):
        """Search decks.

        1. Include card(s) to search for
        !crdatae search fb log

        2. Exclude card(s) to search for (use - as prefix)
        !crdatae search golem -lightning

        3. Elixir range (add elixir=min-max)
        !crdatae search hog elixir=0-3.2

        e.g.: Find 3M Hog decks without battle ram under 4 elixir
        !crdatae search 3m hog -br elixir=0-4
        """
        if not len(cards):
            await self.bot.say("You must enter at least one card.")
            await send_cmd_help(ctx)
            return

        crdata = self.bot.get_cog('CRData')

        if crdata is None:
            await self.bot.say(self.error_msg["requires_crdata"])
            return

        found_decks = await crdata.search(ctx, *cards)

        if found_decks is None:
            await self.bot.say("Found 0 decks.")
            return
        if not len(found_decks):
            await self.bot.say("Found 0 decks.")
            return

        decks = []
        for fd in found_decks:
            card_keys = [crdata.sfid_to_id(card["key"]) for card in fd["deck"]]
            card_levels = [card["level"] for card in fd["deck"]]
            deck = Deck(card_keys=card_keys, card_levels=card_levels, rank=fd["ranks"][0], usage=fd["count"])
            decks.append(deck)

        per_page = 25
        decks_group = list(grouper(per_page, decks))
        color = random_discord_color()

        for page, decks_page in enumerate(decks_group):

            em = self.embed_decks_search(
                decks_page,
                page=(page + 1),
                title="Clash Royale: Global Top 200 Decks",
                description="Found {} decks.".format(len(decks)),
                color=color,
                footer_text="Data provided by http://starfi.re",
                show_usage=False
            )

            await self.bot.say(embed=em)

            if page < len(decks_group) - 1:
                show_next = await self.show_next_page(ctx)
                if not show_next:
                    await self.bot.say("Search results aborted.")
                    break

    def embed_decks_search(self, decks, **kwargs):
        """Show embed decks.

        Params:
        + page. Current page.
        + per_page. Number of results per page.
        + All parameters supported by Discord Embeds.
        """
        em = discord.Embed(**kwargs)

        page = kwargs.get('page', 1)
        per_page = kwargs.get('per_page', 25)
        show_usage = kwargs.get('show_usage', False)
        footer_text = kwargs.get('footer_text', '')

        for deck_id, deck in enumerate(decks):
            if deck is not None:
                result_number = per_page * (page - 1) + (deck_id + 1)

                usage_str = ''
                if deck.usage and show_usage:
                    usage_str = '(Usage: {})'.format(deck.usage)
                field_name = "{}: Rank {} {}".format(result_number, deck.rank, usage_str)
                field_value = '{}\n{}'.format(
                    deck.emoji_repr(self.be, show_levels=True),
                    deck.avg_elixir_str)
                em.add_field(name=field_name, value=field_value)

        em.set_footer(text=footer_text)
        return em

    async def show_next_page(self, ctx):
        """Results pagination."""
        timeout = 30
        await self.bot.say(
            "Would you like to see more results? (y/n)")
        answer = await self.bot.wait_for_message(
            timeout=timeout,
            author=ctx.message.author)
        if answer is None:
            return False
        elif not len(answer.content):
            return False
        elif answer.content[0].lower() != 'y':
            return False
        return True

    def card_elixir(self, card):
        """Return elixir of a card."""
        elixir = 0
        try:
            elixir = self.clashroyale["Cards"][card]["elixir"]
        except KeyError:
            pass
        return elixir

def setup(bot):
    """Setup bot."""
    n = CRDataEnhanced(bot)
    bot.add_cog(n)
<commit_msg>[crdatae] changed deck per page from 25 to 10 — discord appeared to have lower the embed size limit. this solves the issue<commit_after>"""
The MIT License (MIT)

Copyright (c) 2017 SML

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""

import itertools
import os
from random import choice

import discord
from __main__ import send_cmd_help
from discord.ext import commands

from cogs.utils.dataIO import dataIO

PATH = os.path.join("data", "crdatae")
CLASH_ROYALE_JSON = os.path.join(PATH, "clashroyale.json")

def grouper(n, iterable, fillvalue=None):
    """Group lists into lists of items.

    grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx"""
    args = [iter(iterable)] * n
    return itertools.zip_longest(*args, fillvalue=fillvalue)

def random_discord_color():
    """Return random color as an integer."""
    color = ''.join([choice('0123456789ABCDEF') for x in range(6)])
    color = int(color, 16)
    return discord.Color(value=color)

class BotEmoji:
    """Emojis available in bot."""
    def __init__(self, bot):
        self.bot = bot

    def name(self, name):
        """Emoji by name."""
        for server in self.bot.servers:
            for emoji in server.emojis:
                if emoji.name == name:
                    return '<:{}:{}>'.format(emoji.name, emoji.id)
        return ''

    def key(self, key):
        """Chest emojis by api key name or key.

        name is used by this cog.
        key is values returned by the api.
        Use key only if name is not set
        """
        if key in self.map:
            name = self.map[key]
            return self.name(name)
        return ''

class ClashRoyale:
    """Clash Royale Data."""
    instance = None

    class __ClashRoyale:
        """Singleton."""
        def __init__(self, *args, **kwargs):
            """Init."""
            self.data = dataIO.load_json(CLASH_ROYALE_JSON)

    def __init__(self, *args, **kwargs):
        """Init."""
        if not ClashRoyale.instance:
            ClashRoyale.instance = ClashRoyale.__ClashRoyale(*args, **kwargs)
        else:
            pass

    def __getattr__(self, name):
        return getattr(self.instance, name)

    def card_elixir(self, card):
        """"Elixir of a card."""
        try:
            return self.data["Cards"][card]["elixir"]
        except KeyError:
            return 0

class Card():
    """Clash Royale Card."""

    def __init__(self, key=None, level=None):
        """Init.

        Params
        + name (str). Key in the ClashRoyale.json
        """
        self.key = key
        self.level = level

    @property
    def elixir(self):
        """Elixir value."""
        return ClashRoyale().card_elixir(self.key)

    def emoji(self, be: BotEmoji):
        """Emoji representation of the card."""
        if self.key is None:
            return ''
        name = self.key.replace('-', '')
        return be.name(name)

class Deck():
    """Clash Royale Deck.

    Contains 8 cards.
    """

    def __init__(self, card_keys=None, card_levels=None, rank=0, usage=0):
        """Init.

        Params
        + rank (int). Rank on the leaderboard.
        + cards []. List of card ids (keys in ClashRoyale.json).
        + card_levels []. List of card levels.
        """
        self.rank = rank
        self.usage = usage
        self.cards = [Card(key=key) for key in card_keys]
        if card_levels is not None:
            kl_zip = zip(card_keys, card_levels)
            self.cards = [Card(key=k, level=l) for k, l in kl_zip]

    @property
    def avg_elixir(self):
        """Average elixir of the deck."""
        elixirs = [c.elixir for c in self.cards if c.elixir != 0]
        return sum(elixirs) / len(elixirs)

    @property
    def avg_elixir_str(self):
        """Average elixir with format."""
        return 'Average Elixir: {:.3}'.format(self.avg_elixir)

    def emoji_repr(self, be: BotEmoji, show_levels=False):
        """Emoji representaion."""
        out = []
        for card in self.cards:
            emoji = card.emoji(be)
            level = card.level
            level_str = ''
            if show_levels and level is not None:
                level_str = '`{:.<2}`'.format(level)
            out.append('{}{}'.format(emoji, level_str))
        return ''.join(out)

    def __repr__(self):
        return ' '.join([c.key for c in self.cards])

class CRDataEnhanced:
    """Clash Royale Data - Enchanced options.

    Requires CRData cog to function.
    """

    error_msg = {
        "requires_crdata": (
            "The CRData cog is not installed or loaded. "
            "This cog cannot function without it."
        )
    }

    def __init__(self, bot):
        """Init."""
        self.bot = bot
        self.be = BotEmoji(bot)
        self.clashroyale = ClashRoyale().data
        self.per_page = 10

    @commands.group(pass_context=True, no_pm=True)
    async def crdatae(self, ctx):
        """Clash Royale Real-Time Global 200 Leaderboard."""
        if ctx.invoked_subcommand is None:
            await send_cmd_help(ctx)

    @crdatae.command(name="leaderboard", aliases=['lb'], pass_context=True, no_pm=True)
    async def crdatae_leaderboard(self, ctx):
        """Leaderboard."""
        crdata = self.bot.get_cog('CRData')
        if crdata is None:
            await self.bot.say(self.error_msg["requires_crdata"])
            return

        data = crdata.get_last_data()
        # decks = data["decks"]

        decks = []
        for rank, deck in enumerate(data["decks"], 1):
            cards = [crdata.sfid_to_id(card["key"]) for card in deck]
            levels = [card["level"] for card in deck]
            decks.append(Deck(card_keys=cards, card_levels=levels, rank=rank))

        # embeds
        per_page = self.per_page
        decks_group = list(grouper(per_page, decks))
        color = random_discord_color()

        for em_id, decks in enumerate(decks_group):

            em = self.embed_decks_leaderboard(
                decks,
                page=(em_id + 1),
                title="Clash Royale: Global Top 200 Decks",
                color=color,
                footer_text="Data provided by http://starfi.re"
            )

            await self.bot.say(embed=em)

            if em_id < len(decks_group) - 1:
                show_next = await self.show_next_page(ctx)
                if not show_next:
                    await self.bot.say("Search results aborted.")
                    break

    def embed_decks_leaderboard(self, decks, **kwargs):
        """Show embed decks.

        Params:
        + page. Current page.
        + per_page. Number of results per page.
        + All parameters supported by Discord Embeds.
        """
        em = discord.Embed(**kwargs)

        page = kwargs.get('page', 1)
        per_page = kwargs.get('per_page', self.per_page)
        show_usage = kwargs.get('show_usage', False)
        footer_text = kwargs.get('footer_text', '')

        for deck_id, deck in enumerate(decks):
            if deck is not None:
                usage_str = ''
                if deck.usage and show_usage:
                    usage_str = '(Usage: {})'.format(deck.usage)
                field_name = "Rank {} {}".format(deck.rank, usage_str)
                field_value = '{}\n{}'.format(
                    deck.emoji_repr(self.be, show_levels=True),
                    deck.avg_elixir_str)
                em.add_field(name=field_name, value=field_value)

        em.set_footer(text=footer_text)
        return em

    @crdatae.command(name="search", pass_context=True, no_pm=True)
    async def crdatae_search(self, ctx, *cards):
        """Search decks.

        1. Include card(s) to search for
        !crdatae search fb log

        2. Exclude card(s) to search for (use - as prefix)
        !crdatae search golem -lightning

        3. Elixir range (add elixir=min-max)
        !crdatae search hog elixir=0-3.2

        e.g.: Find 3M Hog decks without battle ram under 4 elixir
        !crdatae search 3m hog -br elixir=0-4
        """
        if not len(cards):
            await self.bot.say("You must enter at least one card.")
            await send_cmd_help(ctx)
            return

        crdata = self.bot.get_cog('CRData')

        if crdata is None:
            await self.bot.say(self.error_msg["requires_crdata"])
            return

        found_decks = await crdata.search(ctx, *cards)

        if found_decks is None:
            await self.bot.say("Found 0 decks.")
            return
        if not len(found_decks):
            await self.bot.say("Found 0 decks.")
            return

        decks = []
        for fd in found_decks:
            card_keys = [crdata.sfid_to_id(card["key"]) for card in fd["deck"]]
            card_levels = [card["level"] for card in fd["deck"]]
            deck = Deck(card_keys=card_keys, card_levels=card_levels, rank=fd["ranks"][0], usage=fd["count"])
            decks.append(deck)

        per_page = self.per_page
        decks_group = list(grouper(per_page, decks))
        color = random_discord_color()

        for page, decks_page in enumerate(decks_group):

            em = self.embed_decks_search(
                decks_page,
                page=(page + 1),
                title="Clash Royale: Global Top 200 Decks",
                description="Found {} decks.".format(len(decks)),
                color=color,
                footer_text="Data provided by http://starfi.re",
                show_usage=False
            )

            await self.bot.say(embed=em)

            if page < len(decks_group) - 1:
                show_next = await self.show_next_page(ctx)
                if not show_next:
                    await self.bot.say("Search results aborted.")
                    break

    def embed_decks_search(self, decks, **kwargs):
        """Show embed decks.

        Params:
        + page. Current page.
        + per_page. Number of results per page.
        + All parameters supported by Discord Embeds.
        """
        em = discord.Embed(**kwargs)

        page = kwargs.get('page', 1)
        per_page = kwargs.get('per_page', self.per_page)
        show_usage = kwargs.get('show_usage', False)
        footer_text = kwargs.get('footer_text', '')

        for deck_id, deck in enumerate(decks):
            if deck is not None:
                result_number = per_page * (page - 1) + (deck_id + 1)

                usage_str = ''
                if deck.usage and show_usage:
                    usage_str = '(Usage: {})'.format(deck.usage)
                field_name = "{}: Rank {} {}".format(result_number, deck.rank, usage_str)
                field_value = '{}\n{}'.format(
                    deck.emoji_repr(self.be, show_levels=True),
                    deck.avg_elixir_str)
                em.add_field(name=field_name, value=field_value)

        em.set_footer(text=footer_text)
        return em

    async def show_next_page(self, ctx):
        """Results pagination."""
        timeout = 30
        await self.bot.say(
            "Would you like to see more results? (y/n)")
        answer = await self.bot.wait_for_message(
            timeout=timeout,
            author=ctx.message.author)
        if answer is None:
            return False
        elif not len(answer.content):
            return False
        elif answer.content[0].lower() != 'y':
            return False
        return True

    def card_elixir(self, card):
        """Return elixir of a card."""
        elixir = 0
        try:
            elixir = self.clashroyale["Cards"][card]["elixir"]
        except KeyError:
            pass
        return elixir

def setup(bot):
    """Setup bot."""
    n = CRDataEnhanced(bot)
    bot.add_cog(n)
<|endoftext|>


<commit_before>#!/bin/python

import os, functools, subprocess, sys

def target_sort(a, b):
    if a == b:
        return 0

    # all target is first
    if a == 'all':
        return -1
    if b == 'all':
        return 1

    # .PHONY is last
    if a == '.PHONY':
        return 1
    if b == '.PHONY':
        return -1

    # Sort targets creating files after meta-targets
    if a[0] == '$':
        if b[0] != '$':
            return 1
    elif b[0] == '$':
        return -1

    if a < b:
        return -1
    if a > b:
        return 1

def all_targets_so_far(depends, operations):
    return sorted(list(set(list(depends.keys()) + list(operations.keys()))), key=functools.cmp_to_key(target_sort))

if len(sys.argv) == 1 or sys.argv[1] not in ['gcc', 'clang']:
    print('Specify a compiler ("gcc" or "clang")')
    sys.exit(1)

program_name = 'genetic_chess'
final_targets = ["release", "debug"]
depends = dict()
depends['all'] = final_targets + ["docs"]
depends['clean'] = (f"clean_{target}" for target in depends['all'])
depends["docs"] = ["user_manual", "code_docs"]
depends['clean_docs'] = (f"clean_{target}" for target in depends['docs'])

options = dict()
obj_dest = dict()
bin_dest = dict()
operations = dict()
bins = dict()
link_dirs = dict()

depends["code_docs"] = ['$(DOC_INDEX)']
operations['$(DOC_INDEX)'] = ['doxygen']
depends['$(DOC_INDEX)'] = ["Doxyfile", "$(ALL_SOURCES)"]

user_manual = 'doc/reference.pdf'
user_manual_tex = f'{os.path.splitext(user_manual)[0]}.tex'
user_manual_var = '$(USER_MANUAL)'
depends["user_manual"] = [user_manual_var]
depends[user_manual_var] = [
    user_manual_tex,
    'gene_pool_config_example.txt',
    'genetic_ai_example.txt',
    'doc/game-endings-log-plot.png',
    'doc/game_length_distribution.png',
    'doc/game_length_log_norm_distribution.png',
    'doc/pawn-crash-strength-plot.png',
    'doc/piece-strength-with-king-plot.png',
    'doc/win-lose-plot.png']
operations[user_manual_var] = [f'latexmk -synctex=1 -pdf -cd {user_manual_tex}']

for target in final_targets:
    options[target] = f"$(CFLAGS_{target.upper()}) $(LDFLAGS_{target.upper()})"

    obj_dest[target]= f"$({target.upper()}_OBJ_DIR)"
    bin_dest[target]= f"$({target.upper()}_BIN_DIR)"

    out_variable = f"$(OUT_{target.upper()})"
    link_dir_variable = f"$(LINK_DIR_{target.upper()})"
    all_objects = f"$(OBJ_{target.upper()})"
    operations[out_variable] = [f"mkdir -p {bin_dest[target]}",
                                ' '.join(["$(LD)",
                                          "-o", out_variable,
                                          all_objects,
                                          "$(LDFLAGS)",
                                          f"$(LDFLAGS_{target.upper()})",
                                          "$(CFLAGS)",
                                          f"$(CFLAGS_{target.upper()})"])]
    link_dirs[target] = os.path.join('bin', target)
    operations[os.path.join(link_dir_variable, '$(BIN)')] = [f'mkdir -p {link_dir_variable}',
                                                             f'ln -sf -t {link_dir_variable} `realpath {out_variable}`',
                                                             f'touch {out_variable}']
    depends[os.path.join(link_dir_variable, '$(BIN)')] = [out_variable, 'Makefile']
    bins[target] = os.path.join(bin_dest[target], '$(BIN)')

    depends[target] = [out_variable, os.path.join(link_dir_variable, '$(BIN)')]
    depends[out_variable] = [all_objects]

    operations[f'clean_{target}'] = [f"rm -rf {obj_dest[target]} {bin_dest[target]}"]

    depends[f'test_{target}'] = [target]
    operations[f'test_{target}'] = [f'{bins[target]} -{opt}' for opt in ['test', 'perft', 'speed']]

operations['clean_code_docs'] = ['rm -rf $(DOC_DIR)']
operations['clean_user_manual'] = [f'latexmk -C -cd {user_manual_tex}']
depends['test_all'] = [f'test_{x}' for x in final_targets]
depends['.PHONY'] = [t for t in all_targets_so_far(depends, operations) if not t.startswith('$')]

options_list = dict()
linker_options = dict()
base_options = [
        "-std=c++17",
        "-lstdc++fs",
        "-fno-rtti",
        "-Wshadow",
        "-Wcast-align",
        "-Wundef",
        "-Wfloat-equal",
        "-Wunreachable-code",
        "-pedantic",
        "-Wextra",
        "-Wall",
        "-Werror",
        "-Iinclude"]
base_linker_options = ["-pthread"]
linker_options['debug'] = []
linker_options['release'] = ['-flto']
options_list['debug'] = ["-g"]
options_list['release'] = ["-O3", "-DNDEBUG"]

system = sys.argv[1]
if system == 'gcc':
    compiler = 'g++'
    base_options.extend([
        "-Wzero-as-null-pointer-constant",
        "-Wmain"])
elif system == 'clang':
    compiler = 'clang++'
    linker_options['debug'] = ["-fsanitize=undefined", "-fsanitize=integer"]
    options_list['debug'].extend(["-Og"] + linker_options['debug'])
    base_options.extend([
        "-Wno-unused-command-line-argument",
        "-Wnon-virtual-dtor",
        "-Wredundant-decls",
        "-Wmissing-declarations",
        "-Wmissing-include-dirs",
        "-Wunused-exception-parameter",
        "-Wswitch"])

all_sources = []
for target in final_targets:
    for (dirpath, dirnames, filenames) in os.walk(os.getcwd()):
        dirpath = os.path.relpath(dirpath)
        if dirpath == '.': dirpath = ''
        for source_file in [os.path.join(dirpath, fn) for fn in filenames]:
            if source_file.endswith('.h') or source_file.endswith('.cpp'):
                all_sources.append(source_file)
            if not source_file.endswith('.cpp'):
                continue
            obj_file = os.path.join(obj_dest[target], f"{os.path.splitext(source_file)[0]}.o")
            operations[obj_file] = [f"mkdir -p {os.path.dirname(obj_file)}",
                                    f"$(CXX) $(CFLAGS) $(LDFLAGS) {options[target]} -c {source_file} -o {obj_file}"]

            compiler_command = [compiler] + base_options + options_list[target] + ['-MM', source_file]
            compile_depends = subprocess.check_output(compiler_command).decode('ascii').split(':', 1)[1].split()
            depends[obj_file] = list(set(d for d in compile_depends if d != '\\'))

with open("Makefile", 'w') as make_file:
    make_file.write(f"BIN = {program_name}\n")
    make_file.write(f"CXX = {compiler}\n")
    make_file.write(f"LD = {compiler}\n\n")
    make_file.write(f"CFLAGS = {' '.join(sorted(base_options, key=lambda s : s.lower()))}\n")
    make_file.write(f"LDFLAGS = {' '.join(base_linker_options)}\n\n")
    make_file.write(f"DOC_DIR = {os.path.join('doc', 'doxygen', 'html')}\n")
    make_file.write(f"DOC_INDEX = {os.path.join('$(DOC_DIR)', 'index.html')}\n")
    make_file.write(f"ALL_SOURCES = {' '.join(all_sources)}\n\n")
    make_file.write(f"USER_MANUAL = {user_manual}\n\n")
    for target in final_targets:
        make_file.write(f"{target.upper()}_BIN_DIR = bin/{system}/{target}\n")
        make_file.write(f"OUT_{target.upper()} = {bins[target]}\n")
        make_file.write(f"LINK_DIR_{target.upper()} = {link_dirs[target]}\n")
        make_file.write(f"{target.upper()}_OBJ_DIR = obj/{system}/{target}\n")
        make_file.write(f"OBJ_{target.upper()} = ")
        make_file.write(' '.join([x for x in all_targets_so_far(depends, operations) if x.endswith('.o') and x.startswith(f"$({target.upper()}")]))
        make_file.write('\n')
        make_file.write(f"CFLAGS_{target.upper()} = {' '.join(options_list[target])}\n")
        make_file.write(f"LDFLAGS_{target.upper()} = {' '.join(linker_options[target])}\n\n")

    for target in all_targets_so_far(depends, operations):
        make_file.write(f"{target} : {' '.join(depends.setdefault(target, []))}\n")
        make_file.write("\n".join([f"\t{x}" for x in operations.setdefault(target, [])]))
        make_file.write('\n\n')
<commit_msg>No need to modify dicts in create_Makefile.py<commit_after>#!/bin/python

import os, functools, subprocess, sys

def target_sort(a, b):
    if a == b:
        return 0

    # all target is first
    if a == 'all':
        return -1
    if b == 'all':
        return 1

    # .PHONY is last
    if a == '.PHONY':
        return 1
    if b == '.PHONY':
        return -1

    # Sort targets creating files after meta-targets
    if a[0] == '$':
        if b[0] != '$':
            return 1
    elif b[0] == '$':
        return -1

    if a < b:
        return -1
    if a > b:
        return 1

def all_targets_so_far(depends, operations):
    return sorted(list(set(list(depends.keys()) + list(operations.keys()))), key=functools.cmp_to_key(target_sort))

if len(sys.argv) == 1 or sys.argv[1] not in ['gcc', 'clang']:
    print('Specify a compiler ("gcc" or "clang")')
    sys.exit(1)

program_name = 'genetic_chess'
final_targets = ["release", "debug"]
depends = dict()
depends['all'] = final_targets + ["docs"]
depends['clean'] = (f"clean_{target}" for target in depends['all'])
depends["docs"] = ["user_manual", "code_docs"]
depends['clean_docs'] = (f"clean_{target}" for target in depends['docs'])

options = dict()
obj_dest = dict()
bin_dest = dict()
operations = dict()
bins = dict()
link_dirs = dict()

depends["code_docs"] = ['$(DOC_INDEX)']
operations['$(DOC_INDEX)'] = ['doxygen']
depends['$(DOC_INDEX)'] = ["Doxyfile", "$(ALL_SOURCES)"]

user_manual = 'doc/reference.pdf'
user_manual_tex = f'{os.path.splitext(user_manual)[0]}.tex'
user_manual_var = '$(USER_MANUAL)'
depends["user_manual"] = [user_manual_var]
depends[user_manual_var] = [
    user_manual_tex,
    'gene_pool_config_example.txt',
    'genetic_ai_example.txt',
    'doc/game-endings-log-plot.png',
    'doc/game_length_distribution.png',
    'doc/game_length_log_norm_distribution.png',
    'doc/pawn-crash-strength-plot.png',
    'doc/piece-strength-with-king-plot.png',
    'doc/win-lose-plot.png']
operations[user_manual_var] = [f'latexmk -synctex=1 -pdf -cd {user_manual_tex}']

for target in final_targets:
    options[target] = f"$(CFLAGS_{target.upper()}) $(LDFLAGS_{target.upper()})"

    obj_dest[target]= f"$({target.upper()}_OBJ_DIR)"
    bin_dest[target]= f"$({target.upper()}_BIN_DIR)"

    out_variable = f"$(OUT_{target.upper()})"
    link_dir_variable = f"$(LINK_DIR_{target.upper()})"
    all_objects = f"$(OBJ_{target.upper()})"
    operations[out_variable] = [f"mkdir -p {bin_dest[target]}",
                                ' '.join(["$(LD)",
                                          "-o", out_variable,
                                          all_objects,
                                          "$(LDFLAGS)",
                                          f"$(LDFLAGS_{target.upper()})",
                                          "$(CFLAGS)",
                                          f"$(CFLAGS_{target.upper()})"])]
    link_dirs[target] = os.path.join('bin', target)
    operations[os.path.join(link_dir_variable, '$(BIN)')] = [f'mkdir -p {link_dir_variable}',
                                                             f'ln -sf -t {link_dir_variable} `realpath {out_variable}`',
                                                             f'touch {out_variable}']
    depends[os.path.join(link_dir_variable, '$(BIN)')] = [out_variable, 'Makefile']
    bins[target] = os.path.join(bin_dest[target], '$(BIN)')

    depends[target] = [out_variable, os.path.join(link_dir_variable, '$(BIN)')]
    depends[out_variable] = [all_objects]

    operations[f'clean_{target}'] = [f"rm -rf {obj_dest[target]} {bin_dest[target]}"]

    depends[f'test_{target}'] = [target]
    operations[f'test_{target}'] = [f'{bins[target]} -{opt}' for opt in ['test', 'perft', 'speed']]

operations['clean_code_docs'] = ['rm -rf $(DOC_DIR)']
operations['clean_user_manual'] = [f'latexmk -C -cd {user_manual_tex}']
depends['test_all'] = [f'test_{x}' for x in final_targets]
depends['.PHONY'] = [t for t in all_targets_so_far(depends, operations) if not t.startswith('$')]

options_list = dict()
linker_options = dict()
base_options = [
        "-std=c++17",
        "-lstdc++fs",
        "-fno-rtti",
        "-Wshadow",
        "-Wcast-align",
        "-Wundef",
        "-Wfloat-equal",
        "-Wunreachable-code",
        "-pedantic",
        "-Wextra",
        "-Wall",
        "-Werror",
        "-Iinclude"]
base_linker_options = ["-pthread"]
linker_options['debug'] = []
linker_options['release'] = ['-flto']
options_list['debug'] = ["-g"]
options_list['release'] = ["-O3", "-DNDEBUG"]

system = sys.argv[1]
if system == 'gcc':
    compiler = 'g++'
    base_options.extend([
        "-Wzero-as-null-pointer-constant",
        "-Wmain"])
elif system == 'clang':
    compiler = 'clang++'
    linker_options['debug'] = ["-fsanitize=undefined", "-fsanitize=integer"]
    options_list['debug'].extend(["-Og"] + linker_options['debug'])
    base_options.extend([
        "-Wno-unused-command-line-argument",
        "-Wnon-virtual-dtor",
        "-Wredundant-decls",
        "-Wmissing-declarations",
        "-Wmissing-include-dirs",
        "-Wunused-exception-parameter",
        "-Wswitch"])

all_sources = []
for target in final_targets:
    for (dirpath, dirnames, filenames) in os.walk(os.getcwd()):
        dirpath = os.path.relpath(dirpath)
        if dirpath == '.': dirpath = ''
        for source_file in [os.path.join(dirpath, fn) for fn in filenames]:
            if source_file.endswith('.h') or source_file.endswith('.cpp'):
                all_sources.append(source_file)
            if not source_file.endswith('.cpp'):
                continue
            obj_file = os.path.join(obj_dest[target], f"{os.path.splitext(source_file)[0]}.o")
            operations[obj_file] = [f"mkdir -p {os.path.dirname(obj_file)}",
                                    f"$(CXX) $(CFLAGS) $(LDFLAGS) {options[target]} -c {source_file} -o {obj_file}"]

            compiler_command = [compiler] + base_options + options_list[target] + ['-MM', source_file]
            compile_depends = subprocess.check_output(compiler_command).decode('ascii').split(':', 1)[1].split()
            depends[obj_file] = list(set(d for d in compile_depends if d != '\\'))

with open("Makefile", 'w') as make_file:
    make_file.write(f"BIN = {program_name}\n")
    make_file.write(f"CXX = {compiler}\n")
    make_file.write(f"LD = {compiler}\n\n")
    make_file.write(f"CFLAGS = {' '.join(sorted(base_options, key=lambda s : s.lower()))}\n")
    make_file.write(f"LDFLAGS = {' '.join(base_linker_options)}\n\n")
    make_file.write(f"DOC_DIR = {os.path.join('doc', 'doxygen', 'html')}\n")
    make_file.write(f"DOC_INDEX = {os.path.join('$(DOC_DIR)', 'index.html')}\n")
    make_file.write(f"ALL_SOURCES = {' '.join(all_sources)}\n\n")
    make_file.write(f"USER_MANUAL = {user_manual}\n\n")
    for target in final_targets:
        make_file.write(f"{target.upper()}_BIN_DIR = bin/{system}/{target}\n")
        make_file.write(f"OUT_{target.upper()} = {bins[target]}\n")
        make_file.write(f"LINK_DIR_{target.upper()} = {link_dirs[target]}\n")
        make_file.write(f"{target.upper()}_OBJ_DIR = obj/{system}/{target}\n")
        make_file.write(f"OBJ_{target.upper()} = ")
        make_file.write(' '.join([x for x in all_targets_so_far(depends, operations) if x.endswith('.o') and x.startswith(f"$({target.upper()}")]))
        make_file.write('\n')
        make_file.write(f"CFLAGS_{target.upper()} = {' '.join(options_list[target])}\n")
        make_file.write(f"LDFLAGS_{target.upper()} = {' '.join(linker_options[target])}\n\n")

    for target in all_targets_so_far(depends, operations):
        make_file.write(f"{target} : {' '.join(depends.get(target, []))}\n")
        make_file.write("\n".join([f"\t{x}" for x in operations.get(target, [])]))
        make_file.write('\n\n')
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
pipeline

data pipeline from image root folder to processed tensors of train test batches
for images and labels
"""
import os
import functools
import collections

import numpy as np
import pandas as pd
import tensorflow as tf

from sklearn import model_selection, preprocessing

def folder_traverse(root_dir, ext=None):
    """recusively map all image files from root directory"""

    if not os.path.exists(root_dir):
        raise RuntimeError('{0} doesn\'t exist.'.format(root_dir))

    file_structure = collections.defaultdict(list)
    for item in os.scandir(root_dir):
        if item.is_dir():
            file_structure.update(folder_traverse(item.path, ext))
        elif item.is_file() and item.name.endswith(ext):
            file_structure[os.path.dirname(item.path)].append(item.name)
    return file_structure

def resample(feature_index, labels, balance='auto'):
    """use oversampling to balance class, after split of training set."""

    from imblearn.over_sampling import RandomOverSampler

    ros = RandomOverSampler(ratio=balance)
    feature_index = np.array(feature_index).reshape(-1, 1)
    resampled_index, _ = ros.fit_sample(feature_index, labels)
    resampled_index = [i for nested in resampled_index for i in nested]
    return resampled_index

def generate_data_skeleton(root_dir,
                           ext=None,
                           valid_size=None,
                           oversample=False):
    """turn file structure into human-readable pandas dataframe"""
    file_structure = folder_traverse(root_dir, ext=ext)
    reversed_fs = {k + '/' + f: os.path.splitext(f)[0]
                   for k, v in file_structure.items() for f in v}

    # find the first csv and load it in memory and remove it from dictionary
    for key in reversed_fs:
        if key.endswith('.csv'):
            df_csv = pd.read_csv(key, dtype=np.str)
            reversed_fs.pop(key)
            break

    df = pd.DataFrame.from_dict(data=reversed_fs, orient='index').reset_index()
    df.rename(columns={'index': 'path_to_file', 0: 'filename'}, inplace=True)
    df.reset_index(inplace=True, drop=True)

    df = df_csv.merge(right=df,
                      how='left',
                      left_on='image_name',
                      right_on='filename').dropna(axis=0)

    discrete_labels = [string.split(' ') for string in df['tags'].tolist()]
    mlb = preprocessing.MultiLabelBinarizer()
    mlb.fit(discrete_labels)

    X = np.array(df['path_to_file'])
    y = mlb.transform(discrete_labels)
    X_codified = df['path_to_file'].index
    y_codified = pd.Categorical(df['tags']).codes

    if valid_size:
        print('tags one-hot encoded: \n{0}'.format(mlb.classes_))

        X_train_codified, X_valid_codified, y_train_codified,\
            y_valid_codified = model_selection.train_test_split(
                                X_codified,
                                y_codified,
                                test_size=valid_size)

        if oversample:
            resampled_train_idx = resample(X_train_codified, y_train_codified)
            resampled_valid_idx = resample(X_valid_codified, y_valid_codified)

            X_train, y_train = X[resampled_train_idx], y[resampled_train_idx]
            X_valid, y_valid = X[resampled_valid_idx], y[resampled_valid_idx]
            print('To balance classes, training data has been oversampled'
                  ' to: {0}'.format(len(resampled_train_idx) +
                                    len(resampled_valid_idx)))
        elif not oversample:
            X_train, y_train = X[X_train_codified], y[X_train_codified]
            X_valid, y_valid = X[X_valid_codified], y[X_valid_codified]

        print('training: {0} samples; validation: {1} samples.'.format(
                                        X_train.shape[0], X_valid.shape[0]))
        return X_train, y_train, X_valid, y_valid
    elif not valid_size:
        print('test: {0} samples.'.format(X.shape[0]))
        return X, y

def make_queue(paths_to_image, labels, num_epochs=None, shuffle=True):
    """returns an Ops Tensor with queued image and label pair"""
    images = tf.convert_to_tensor(paths_to_image, dtype=tf.string)
    labels = tf.convert_to_tensor(labels, dtype=tf.float32)
    input_queue = tf.train.slice_input_producer(
                                tensor_list=[images, labels],
                                num_epochs=num_epochs,
                                shuffle=shuffle)
    return input_queue

def decode_transform(input_queue,
                     shape=None,
                     standardize=True,
                     augmentation=None):
    """a single decode and transform function that applies standardization with
    mean centralisation."""
    # input_queue allows slicing with 0: path_to_image, 1: encoded label
    label_queue = input_queue[1]
    image_queue = tf.read_file(input_queue[0])

    # !!! decode_jpeg only accepts RGB jpg but not raising error for CMYK
    original_image = tf.image.decode_jpeg(contents=image_queue, channels=0)
    #
    # original_image = tf.cast(tf.image.decode_png(contents=image_queue,
    #                                              channels=0,
    #                                              dtype=tf.uint16),
    #                          tf.int16)
    # crop larger images to 256*256, this func doesn't 'resize'.
    cropped_img = tf.image.resize_image_with_crop_or_pad(
                                image=original_image,
                                target_height=256,
                                target_width=256)

    # resize cropped images to desired shape
    img = tf.image.resize_images(images=cropped_img, size=shape[:2])
    img.set_shape(shape)

    if augmentation:
        img = tf.image.random_flip_up_down(img)
        img = tf.image.random_flip_left_right(img)
        img = tf.image.rot90(img, k=np.random.randint(4))
        # img = tf.image.random_brightness(img, max_delta=30)
        img = tf.image.random_contrast(img, lower=.8, upper=1.2)
        try:
            # saturation only works for 3-channel RGB images
            img = tf.image.random_saturation(img, lower=.4, upper=2)
        except ValueError:
            pass

        img.set_shape(shape)

    # apply standardization
    if standardize:
        img = tf.image.per_image_standardization(img)

    return img, label_queue

def batch_generator(image, label, batch_size=None, shuffle=True, threads=4):
    """turn data queue into batches"""
    if shuffle:
        return tf.train.shuffle_batch(
                                tensors=[image, label],
                                batch_size=batch_size,
                                num_threads=threads,
                                capacity=1e3,
                                min_after_dequeue=200,
                                allow_smaller_final_batch=True)
    elif not shuffle:
        return tf.train.batch(
                                tensors=[image, label],
                                batch_size=batch_size,
                                num_threads=1,
                                # thread number must be one to be unshuffled.
                                capacity=1e3,
                                allow_smaller_final_batch=True)

def data_pipe(paths_to_image,
              labels,
              num_epochs=None,
              batch_size=None,
              shape=None,
              augmentation=None,
              shuffle=True,
              threads=4):
    """so one-in-all from data directory to iterated data feed in batches"""
    preprocessed_image_queue, label_queue = decode_transform(make_queue(
                                paths_to_image,
                                labels,
                                num_epochs=num_epochs,
                                shuffle=shuffle),
                                augmentation=augmentation,
                                shape=shape)
    image_batch, label_batch = batch_generator(
                                preprocessed_image_queue,
                                label_queue,
                                batch_size=batch_size,
                                shuffle=shuffle,
                                threads=threads)
    return image_batch, label_batch

def multithreading(func):
    """decorator using tensorflow threading ability."""
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)
        func_output = func(*args, **kwargs)
        try:
            coord.request_stop()
            coord.join(threads, stop_grace_period_secs=5)
        except (tf.errors.CancelledError, RuntimeError) as e:
            pass
        return func_output
    return wrapper
<commit_msg>test augmentation<commit_after># -*- coding: utf-8 -*-
"""
pipeline

data pipeline from image root folder to processed tensors of train test batches
for images and labels
"""
import os
import functools
import collections

import numpy as np
import pandas as pd
import tensorflow as tf

from sklearn import model_selection, preprocessing

def folder_traverse(root_dir, ext=None):
    """recusively map all image files from root directory"""

    if not os.path.exists(root_dir):
        raise RuntimeError('{0} doesn\'t exist.'.format(root_dir))

    file_structure = collections.defaultdict(list)
    for item in os.scandir(root_dir):
        if item.is_dir():
            file_structure.update(folder_traverse(item.path, ext))
        elif item.is_file() and item.name.endswith(ext):
            file_structure[os.path.dirname(item.path)].append(item.name)
    return file_structure

def resample(feature_index, labels, balance='auto'):
    """use oversampling to balance class, after split of training set."""

    from imblearn.over_sampling import RandomOverSampler

    ros = RandomOverSampler(ratio=balance)
    feature_index = np.array(feature_index).reshape(-1, 1)
    resampled_index, _ = ros.fit_sample(feature_index, labels)
    resampled_index = [i for nested in resampled_index for i in nested]
    return resampled_index

def generate_data_skeleton(root_dir,
                           ext=None,
                           valid_size=None,
                           oversample=False):
    """turn file structure into human-readable pandas dataframe"""
    file_structure = folder_traverse(root_dir, ext=ext)
    reversed_fs = {k + '/' + f: os.path.splitext(f)[0]
                   for k, v in file_structure.items() for f in v}

    # find the first csv and load it in memory and remove it from dictionary
    for key in reversed_fs:
        if key.endswith('.csv'):
            df_csv = pd.read_csv(key, dtype=np.str)
            reversed_fs.pop(key)
            break

    df = pd.DataFrame.from_dict(data=reversed_fs, orient='index').reset_index()
    df.rename(columns={'index': 'path_to_file', 0: 'filename'}, inplace=True)
    df.reset_index(inplace=True, drop=True)

    df = df_csv.merge(right=df,
                      how='left',
                      left_on='image_name',
                      right_on='filename').dropna(axis=0)

    discrete_labels = [string.split(' ') for string in df['tags'].tolist()]
    mlb = preprocessing.MultiLabelBinarizer()
    mlb.fit(discrete_labels)

    X = np.array(df['path_to_file'])
    y = mlb.transform(discrete_labels)
    X_codified = df['path_to_file'].index
    y_codified = pd.Categorical(df['tags']).codes

    if valid_size:
        print('tags one-hot encoded: \n{0}'.format(mlb.classes_))

        X_train_codified, X_valid_codified, y_train_codified,\
            y_valid_codified = model_selection.train_test_split(
                                X_codified,
                                y_codified,
                                test_size=valid_size)

        if oversample:
            resampled_train_idx = resample(X_train_codified, y_train_codified)
            resampled_valid_idx = resample(X_valid_codified, y_valid_codified)

            X_train, y_train = X[resampled_train_idx], y[resampled_train_idx]
            X_valid, y_valid = X[resampled_valid_idx], y[resampled_valid_idx]
            print('To balance classes, training data has been oversampled'
                  ' to: {0}'.format(len(resampled_train_idx) +
                                    len(resampled_valid_idx)))
        elif not oversample:
            X_train, y_train = X[X_train_codified], y[X_train_codified]
            X_valid, y_valid = X[X_valid_codified], y[X_valid_codified]

        print('training: {0} samples; validation: {1} samples.'.format(
                                        X_train.shape[0], X_valid.shape[0]))
        return X_train, y_train, X_valid, y_valid
    elif not valid_size:
        print('test: {0} samples.'.format(X.shape[0]))
        return X, y

def make_queue(paths_to_image, labels, num_epochs=None, shuffle=True):
    """returns an Ops Tensor with queued image and label pair"""
    images = tf.convert_to_tensor(paths_to_image, dtype=tf.string)
    labels = tf.convert_to_tensor(labels, dtype=tf.float32)
    input_queue = tf.train.slice_input_producer(
                                tensor_list=[images, labels],
                                num_epochs=num_epochs,
                                shuffle=shuffle)
    return input_queue

def decode_transform(input_queue,
                     shape=None,
                     standardize=True,
                     augmentation=None):
    """a single decode and transform function that applies standardization with
    mean centralisation."""
    # input_queue allows slicing with 0: path_to_image, 1: encoded label
    label_queue = input_queue[1]
    image_queue = tf.read_file(input_queue[0])

    # !!! decode_jpeg only accepts RGB jpg but not raising error for CMYK
    # original_image = tf.image.decode_jpeg(contents=image_queue, channels=0)

    original_image = tf.cast(tf.image.decode_png(contents=image_queue,
                                                 channels=0,
                                                 dtype=tf.uint16),
                             tf.int16)
    # crop larger images to 256*256, this func doesn't 'resize'.
    cropped_img = tf.image.resize_image_with_crop_or_pad(
                                image=original_image,
                                target_height=256,
                                target_width=256)

    # resize cropped images to desired shape
    img = tf.image.resize_images(images=cropped_img, size=shape[:2])
    img.set_shape(shape)

    if augmentation:
        img = tf.image.random_flip_up_down(img)
        img = tf.image.random_flip_left_right(img)
        img = tf.image.rot90(img, k=np.random.randint(4))
        # img = tf.image.random_brightness(img, max_delta=30)
        img = tf.image.random_contrast(img, lower=.8, upper=1.2)
        try:
            # saturation only works for 3-channel RGB images
            img = tf.image.random_saturation(img, lower=.4, upper=2)
        except ValueError:
            pass

        img.set_shape(shape)

    # apply standardization
    if standardize:
        img = tf.image.per_image_standardization(img)

    return img, label_queue

def batch_generator(image, label, batch_size=None, shuffle=True, threads=4):
    """turn data queue into batches"""
    if shuffle:
        return tf.train.shuffle_batch(
                                tensors=[image, label],
                                batch_size=batch_size,
                                num_threads=threads,
                                capacity=1e3,
                                min_after_dequeue=200,
                                allow_smaller_final_batch=True)
    elif not shuffle:
        return tf.train.batch(
                                tensors=[image, label],
                                batch_size=batch_size,
                                num_threads=1,
                                # thread number must be one to be unshuffled.
                                capacity=1e3,
                                allow_smaller_final_batch=True)

def data_pipe(paths_to_image,
              labels,
              num_epochs=None,
              batch_size=None,
              shape=None,
              augmentation=None,
              shuffle=True,
              threads=4):
    """so one-in-all from data directory to iterated data feed in batches"""
    preprocessed_image_queue, label_queue = decode_transform(make_queue(
                                paths_to_image,
                                labels,
                                num_epochs=num_epochs,
                                shuffle=shuffle),
                                augmentation=augmentation,
                                shape=shape)
    image_batch, label_batch = batch_generator(
                                preprocessed_image_queue,
                                label_queue,
                                batch_size=batch_size,
                                shuffle=shuffle,
                                threads=threads)
    return image_batch, label_batch

def multithreading(func):
    """decorator using tensorflow threading ability."""
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(coord=coord)
        func_output = func(*args, **kwargs)
        try:
            coord.request_stop()
            coord.join(threads, stop_grace_period_secs=5)
        except (tf.errors.CancelledError, RuntimeError) as e:
            pass
        return func_output
    return wrapper
<|endoftext|>