<commit_before>import re
import os.path
import fnmatch

from subprocess import Popen, PIPE

from vial import vfunc, vim
from vial.utils import buffer_with_file, focus_window, vimfunction, \
    get_ws_len, mark, get_key_code, echo, get_projects
from vial.widgets import SearchDialog, ListFormatter, ListView

def escape():
    if len(vim.windows) < 2:
        return

    cur = vfunc.winnr()

    for n, w in reversed(list(enumerate(vim.windows, 1))):
        if not buffer_with_file(w.buffer):
            if not '[Command Line]'in w.buffer.name:
                focus_window(n)
            vim.command('q')
            if n != cur:
                if cur > n:
                    cur -= 1

                focus_window(cur)

            return

def shift_indent(line, shift=1):
    return vfunc.indent(line-1) + shift * vfunc.eval('&sw')

@vimfunction
def indent():
    line, pos = vim.current.window.cursor
    buf = vim.current.buffer
    if buf[line-2].endswith(':'):
        return shift_indent(line)

    pline = buf[line-2].rstrip()
    if pline and pline[-1] in ('(', '[', '{'):
        return shift_indent(line)

    w = vfunc.indent(line-1)
    if not buf[line-1].strip() and w != pos:
        return -1

    return w

search_outline_dialog = None
def search_outline():
    global search_outline_dialog
    if not search_outline_dialog:
        search_outline_dialog = SearchOutlineDialog()

    search_outline_dialog.open()

class SearchOutlineDialog(SearchDialog):
    def __init__(self):
        self.lines = []
        SearchDialog.__init__(self, '__vial_search_outline__',
            ListView(self.lines, ListFormatter(2,0, 3,1)))

    def open(self):
        self.last_window = vfunc.winnr()
        self.sbuf = vim.current.buffer
        self.list_view.clear()
        self.show(u'')
        self.loop.enter()

    def on_select(self, item, cursor):
        focus_window(self.last_window)
        mark()
        vim.command('normal! {}Gzz^'.format(item[1]))

    def on_cancel(self):
        focus_window(self.last_window)

    def on_prompt_changed(self, prompt):
        if prompt:
            self.fill(prompt)
        else:
            self.list_view.show_cursor(False)
            self.buf[0:] = ['Type something to search']
            self.loop.refresh()

    def fill(self, prompt):
        current = self.current = object()
        self.list_view.clear()

        ignorecase = prompt == prompt.lower()
        if ignorecase:
            prompt = prompt.lower()

        for i, line in enumerate(self.sbuf, 1):
            if current is not self.current:
                return

            l = line.decode('utf-8')
            if ignorecase:
                l = l.lower()

            if prompt in l:
                self.lines.append((get_ws_len(line), i, str(i), line))

        self.lines.sort()
        self.list_view.render()
        self.loop.refresh()

def changed_projects():
    changed = []
    for p in get_projects():
        if not os.path.exists(os.path.join(p, '.git')):
            continue

        proc = Popen(['git', 'status', '-b', '--porcelain'], cwd=p, stdout=PIPE, stderr=PIPE)
        out, err = proc.communicate()

        if proc.returncode != 0:
            if err.strip():
                print err
            else:
                print 'Git error', proc.returncode
        else:
            for line in out.splitlines():
                if not line.startswith('##') or 'ahead' in line:
                    changed.append(p)
                    break

    if changed:
        print ', '.join(os.path.basename(r) for r in changed)
    else:
        echo('There are no any changes')

def new(fname):
    dname = os.path.dirname(fname)
    if not os.path.exists(dname):
        os.makedirs(dname)

    vim.command('e {}'.format(fname))

def filter_qf(pattern):
    qf = vfunc.getqflist()
    if not qf:
        return

    regex = re.compile(fnmatch.translate(pattern))

    result = []
    for r in qf:
       fname = vfunc.bufname(r['bufnr'])
       if regex.match(fname):
           nr = dict(r)
           nr['filename'] = fname
           result.append(nr)

    vfunc.setqflist(result)

def add_project(project):
    projects = list(vim.vars.get('vial_projects', []))
    projects.append(project)
    vim.vars['vial_projects'] = projects

def clear_projects():
    vim.vars['vial_projects'] = []
<commit_msg>indent: respect pep8<commit_after>import re
import os.path
import fnmatch

from subprocess import Popen, PIPE

from vial import vfunc, vim
from vial.utils import buffer_with_file, focus_window, vimfunction, \
    get_ws_len, mark, get_key_code, echo, get_projects
from vial.widgets import SearchDialog, ListFormatter, ListView

def escape():
    if len(vim.windows) < 2:
        return

    cur = vfunc.winnr()

    for n, w in reversed(list(enumerate(vim.windows, 1))):
        if not buffer_with_file(w.buffer):
            if not '[Command Line]'in w.buffer.name:
                focus_window(n)
            vim.command('q')
            if n != cur:
                if cur > n:
                    cur -= 1

                focus_window(cur)

            return

def shift_indent(line, shift=1):
    return vfunc.indent(line-1) + shift * vfunc.eval('&sw')

@vimfunction
def indent():
    line, pos = vim.current.window.cursor
    buf = vim.current.buffer
    if buf[line-2].endswith(':'):
        return shift_indent(line)

    pline = buf[line-2].rstrip()
    if pline and pline[-1] in ('(', '[', '{'):
        return shift_indent(line)

    if pline:
        start = -1
        while True:
            start = pline.find('(', start + 1)
            if start < 0: break
            vfunc.cursor(line, start + 1)
            l, p = vfunc.searchpairpos('(', '', ')', 'nW', '', line)
            if not l:
                return start + 1

    w = vfunc.indent(line-1)
    if not buf[line-1].strip() and w != pos:
        return -1

    return w

search_outline_dialog = None
def search_outline():
    global search_outline_dialog
    if not search_outline_dialog:
        search_outline_dialog = SearchOutlineDialog()

    search_outline_dialog.open()

class SearchOutlineDialog(SearchDialog):
    def __init__(self):
        self.lines = []
        SearchDialog.__init__(self, '__vial_search_outline__',
            ListView(self.lines, ListFormatter(2,0, 3,1)))

    def open(self):
        self.last_window = vfunc.winnr()
        self.sbuf = vim.current.buffer
        self.list_view.clear()
        self.show(u'')
        self.loop.enter()

    def on_select(self, item, cursor):
        focus_window(self.last_window)
        mark()
        vim.command('normal! {}Gzz^'.format(item[1]))

    def on_cancel(self):
        focus_window(self.last_window)

    def on_prompt_changed(self, prompt):
        if prompt:
            self.fill(prompt)
        else:
            self.list_view.show_cursor(False)
            self.buf[0:] = ['Type something to search']
            self.loop.refresh()

    def fill(self, prompt):
        current = self.current = object()
        self.list_view.clear()

        ignorecase = prompt == prompt.lower()
        if ignorecase:
            prompt = prompt.lower()

        for i, line in enumerate(self.sbuf, 1):
            if current is not self.current:
                return

            l = line.decode('utf-8')
            if ignorecase:
                l = l.lower()

            if prompt in l:
                self.lines.append((get_ws_len(line), i, str(i), line))

        self.lines.sort()
        self.list_view.render()
        self.loop.refresh()

def changed_projects():
    changed = []
    for p in get_projects():
        if not os.path.exists(os.path.join(p, '.git')):
            continue

        proc = Popen(['git', 'status', '-b', '--porcelain'], cwd=p, stdout=PIPE, stderr=PIPE)
        out, err = proc.communicate()

        if proc.returncode != 0:
            if err.strip():
                print err
            else:
                print 'Git error', proc.returncode
        else:
            for line in out.splitlines():
                if not line.startswith('##') or 'ahead' in line:
                    changed.append(p)
                    break

    if changed:
        print ', '.join(os.path.basename(r) for r in changed)
    else:
        echo('There are no any changes')

def new(fname):
    dname = os.path.dirname(fname)
    if not os.path.exists(dname):
        os.makedirs(dname)

    vim.command('e {}'.format(fname))

def filter_qf(pattern):
    qf = vfunc.getqflist()
    if not qf:
        return

    regex = re.compile(fnmatch.translate(pattern))

    result = []
    for r in qf:
       fname = vfunc.bufname(r['bufnr'])
       if regex.match(fname):
           nr = dict(r)
           nr['filename'] = fname
           result.append(nr)

    vfunc.setqflist(result)

def add_project(project):
    projects = list(vim.vars.get('vial_projects', []))
    projects.append(project)
    vim.vars['vial_projects'] = projects

def clear_projects():
    vim.vars['vial_projects'] = []
<|endoftext|>


<commit_before>#!/usr/bin/env python3

import argparse
import csv
import cv2
import skvideo.io               # pip3 install sk-video
import json
import math
import numpy as np
import os

from props import PropertyNode
import props_json

import sys
sys.path.append('../scripts')
from lib import transformations

import camera
import horizon

# constants
d2r = math.pi / 180.0
r2d = 180.0 / math.pi

parser = argparse.ArgumentParser(description='Estimate gyro biases from movie.')
parser.add_argument('video', help='video file')
parser.add_argument('--camera', help='select camera calibration file')
parser.add_argument('--scale', type=float, default=1.0, help='scale input')
parser.add_argument('--skip-frames', type=int, default=0, help='skip n initial frames')
parser.add_argument('--write', action='store_true', help='write out the final video')
args = parser.parse_args()

#file = args.video
scale = args.scale
skip_frames = args.skip_frames

# pathname work
abspath = os.path.abspath(args.video)
filename, ext = os.path.splitext(abspath)
dirname = os.path.dirname(args.video)
output_csv = filename + "_horiz.csv"
output_video = filename + "_horiz" + ext
local_config = os.path.join(dirname, "camera.json")

camera = camera.VirtualCamera()
camera.load(args.camera, local_config, args.scale)
K = camera.get_K()
IK = camera.get_IK()
dist = camera.get_dist()
print('Camera:', camera.get_name())
print('K:\n', K)
print('dist:', dist)

cu = K[0,2]
cv = K[1,2]

metadata = skvideo.io.ffprobe(args.video)
#print(metadata.keys())
print(json.dumps(metadata["video"], indent=4))
fps_string = metadata['video']['@avg_frame_rate']
(num, den) = fps_string.split('/')
fps = float(num) / float(den)
codec = metadata['video']['@codec_long_name']
w = int(round(int(metadata['video']['@width']) * scale))
h = int(round(int(metadata['video']['@height']) * scale))
total_frames = int(round(float(metadata['video']['@duration']) * fps))

print('fps:', fps)
print('codec:', codec)
print('output size:', w, 'x', h)
print('total frames:', total_frames)

print("Opening ", args.video)
reader = skvideo.io.FFmpegReader(args.video, inputdict={}, outputdict={})

csvfile = open(output_csv, 'w')
fieldnames = [ 'frame', 'video time',
               'camera roll (deg)', 'camera pitch (deg)',
               'roll rate (rad/sec)', 'pitch rate (rad/sec)' ]
csv_writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
csv_writer.writeheader()

inputdict = {
    '-r': str(fps)
}

lossless = {
    # See all options: https://trac.ffmpeg.org/wiki/Encode/H.264
    '-vcodec': 'libx264',  # use the h.264 codec
    '-crf': '0',           # set the constant rate factor to 0, (lossless)
    '-preset': 'veryslow', # maximum compression
    '-r': str(fps)         # match input fps
}

sane = {
    # See all options: https://trac.ffmpeg.org/wiki/Encode/H.264
    '-vcodec': 'libx264',  # use the h.264 codec
    '-crf': '17',          # visually lossless (or nearly so)
    '-preset': 'medium',   # default compression
    '-r': str(fps)         # match input fps
}

video_writer = skvideo.io.FFmpegWriter(output_video, inputdict=inputdict, outputdict=sane)

#frame = cv2.imread("/home/curt/Downloads/Frame1.png")
#frame_scale = cv2.resize(frame, (0,0), fx=scale, fy=scale,
#                         interpolation=cv2.INTER_AREA)
#cv2.imshow('scaled orig', frame_scale)
#frame_undist = cv2.undistort(frame_scale, K, np.array(dist))
#cv2.imshow("undist", frame_undist)
#cv2.waitKey()

counter = 0
last_roll = None
last_pitch = None
for frame in reader.nextFrame():
    frame = frame[:,:,::-1]     # convert from RGB to BGR (to make opencv happy)

    if counter < skip_frames:
        if counter % 100 == 0:
            print("Skipping %d frames..." % counter)
        else:
            continue

    print("Frame %d" % counter)

    method = cv2.INTER_AREA
    frame_scale = cv2.resize(frame, (0,0), fx=scale, fy=scale,
                             interpolation=method)
    cv2.imshow('scaled orig', frame_scale)
    frame_undist = cv2.undistort(frame_scale, K, np.array(dist))

    # test horizon detection
    lines = horizon.horizon(frame_undist)
    if not lines is None:
        #best_line = horizon.track_best(lines)
        best_line = lines[0]
        roll, pitch = horizon.get_camera_attitude(best_line, IK, cu, cv)
        if last_roll is None or last_pitch is None:
            roll_rate = 0
            pitch_rate = 0
        else:
            df = counter - last_counter
            roll_rate = (roll - last_roll) * fps * d2r
            pitch_rate = (pitch - last_pitch) * fps * d2r
        last_counter = counter
        last_roll = roll
        last_pitch = pitch
        horizon.draw(frame_undist, best_line, IK, cu, cv)
        row = {'frame': counter,
               'video time': "%.4f" % (counter / fps),
               'camera roll (deg)': "%.2f" % roll,
               'camera pitch (deg)': "%.2f" % pitch,
               'roll rate (rad/sec)': "%.3f" % roll_rate,
               'pitch rate (rad/sec)': "%.3f" % pitch_rate }
        csv_writer.writerow(row)
    else:
        roll = None
        pitch = None
        roll_rate = 0.0
        pitch_rate = 0.0
        last_counter = None
        last_roll = None
        last_pitch = None

    counter += 1

    cv2.imshow("horizon", frame_undist)
    if args.write:
        #write the frame as RGB not BGR
        video_writer.writeFrame(frame_undist[:,:,::-1])
    if 0xFF & cv2.waitKey(5) == 27:
        break

csvfile.close()
cv2.destroyAllWindows()

<commit_msg>Fiddling with the counter value and when it gets incremented.<commit_after>#!/usr/bin/env python3

import argparse
import csv
import cv2
import skvideo.io               # pip3 install sk-video
import json
import math
import numpy as np
import os

from props import PropertyNode
import props_json

import sys
sys.path.append('../scripts')
from lib import transformations

import camera
import horizon

# constants
d2r = math.pi / 180.0
r2d = 180.0 / math.pi

parser = argparse.ArgumentParser(description='Estimate gyro biases from movie.')
parser.add_argument('video', help='video file')
parser.add_argument('--camera', help='select camera calibration file')
parser.add_argument('--scale', type=float, default=1.0, help='scale input')
parser.add_argument('--skip-frames', type=int, default=0, help='skip n initial frames')
parser.add_argument('--write', action='store_true', help='write out the final video')
args = parser.parse_args()

#file = args.video
scale = args.scale
skip_frames = args.skip_frames

# pathname work
abspath = os.path.abspath(args.video)
filename, ext = os.path.splitext(abspath)
dirname = os.path.dirname(args.video)
output_csv = filename + "_horiz.csv"
output_video = filename + "_horiz" + ext
local_config = os.path.join(dirname, "camera.json")

camera = camera.VirtualCamera()
camera.load(args.camera, local_config, args.scale)
K = camera.get_K()
IK = camera.get_IK()
dist = camera.get_dist()
print('Camera:', camera.get_name())
print('K:\n', K)
print('dist:', dist)

cu = K[0,2]
cv = K[1,2]

metadata = skvideo.io.ffprobe(args.video)
#print(metadata.keys())
print(json.dumps(metadata["video"], indent=4))
fps_string = metadata['video']['@avg_frame_rate']
(num, den) = fps_string.split('/')
fps = float(num) / float(den)
codec = metadata['video']['@codec_long_name']
w = int(round(int(metadata['video']['@width']) * scale))
h = int(round(int(metadata['video']['@height']) * scale))
total_frames = int(round(float(metadata['video']['@duration']) * fps))

print('fps:', fps)
print('codec:', codec)
print('output size:', w, 'x', h)
print('total frames:', total_frames)

print("Opening ", args.video)
reader = skvideo.io.FFmpegReader(args.video, inputdict={}, outputdict={})

csvfile = open(output_csv, 'w')
fieldnames = [ 'frame', 'video time',
               'camera roll (deg)', 'camera pitch (deg)',
               'roll rate (rad/sec)', 'pitch rate (rad/sec)' ]
csv_writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
csv_writer.writeheader()

inputdict = {
    '-r': str(fps)
}

lossless = {
    # See all options: https://trac.ffmpeg.org/wiki/Encode/H.264
    '-vcodec': 'libx264',  # use the h.264 codec
    '-crf': '0',           # set the constant rate factor to 0, (lossless)
    '-preset': 'veryslow', # maximum compression
    '-r': str(fps)         # match input fps
}

sane = {
    # See all options: https://trac.ffmpeg.org/wiki/Encode/H.264
    '-vcodec': 'libx264',  # use the h.264 codec
    '-crf': '17',          # visually lossless (or nearly so)
    '-preset': 'medium',   # default compression
    '-r': str(fps)         # match input fps
}

video_writer = skvideo.io.FFmpegWriter(output_video, inputdict=inputdict, outputdict=sane)

#frame = cv2.imread("/home/curt/Downloads/Frame1.png")
#frame_scale = cv2.resize(frame, (0,0), fx=scale, fy=scale,
#                         interpolation=cv2.INTER_AREA)
#cv2.imshow('scaled orig', frame_scale)
#frame_undist = cv2.undistort(frame_scale, K, np.array(dist))
#cv2.imshow("undist", frame_undist)
#cv2.waitKey()

counter = -1
last_roll = None
last_pitch = None
for frame in reader.nextFrame():
    frame = frame[:,:,::-1]     # convert from RGB to BGR (to make opencv happy)
    counter += 1

    if counter < skip_frames:
        if counter % 100 == 0:
            print("Skipping %d frames..." % counter)
        else:
            continue

    print("Frame %d" % counter)

    method = cv2.INTER_AREA
    frame_scale = cv2.resize(frame, (0,0), fx=scale, fy=scale,
                             interpolation=method)
    cv2.imshow('scaled orig', frame_scale)
    frame_undist = cv2.undistort(frame_scale, K, np.array(dist))

    # test horizon detection
    lines = horizon.horizon(frame_undist)
    if not lines is None:
        #best_line = horizon.track_best(lines)
        best_line = lines[0]
        roll, pitch = horizon.get_camera_attitude(best_line, IK, cu, cv)
        if last_roll is None or last_pitch is None:
            roll_rate = 0
            pitch_rate = 0
        else:
            roll_rate = (roll - last_roll) * fps * d2r
            pitch_rate = (pitch - last_pitch) * fps * d2r
        last_roll = roll
        last_pitch = pitch
        horizon.draw(frame_undist, best_line, IK, cu, cv)
        row = {'frame': counter,
               'video time': "%.4f" % (counter / fps),
               'camera roll (deg)': "%.2f" % roll,
               'camera pitch (deg)': "%.2f" % pitch,
               'roll rate (rad/sec)': "%.3f" % roll_rate,
               'pitch rate (rad/sec)': "%.3f" % pitch_rate }
        csv_writer.writerow(row)
    else:
        roll = None
        pitch = None
        roll_rate = 0.0
        pitch_rate = 0.0
        last_roll = None
        last_pitch = None

    cv2.imshow("horizon", frame_undist)
    if args.write:
        #write the frame as RGB not BGR
        video_writer.writeFrame(frame_undist[:,:,::-1])
    if 0xFF & cv2.waitKey(5) == 27:
        break

csvfile.close()
cv2.destroyAllWindows()

<|endoftext|>


<commit_before># -*- coding: utf8 -*-

import sys
import getopt
import urllib
import webbrowser

from bs4 import BeautifulSoup
import requests

URL_RANDOM = "https://www.metal-archives.com/band/random"
glob_verbose = False
glob_print_page = False

def usage():
  print("     __                 _                   _        _ ")
  print("    /__\ __ _ _ __   __| | ___   /\/\   ___| |_ __ _| |")
  print("   / \/// _` | '_ \ / _` |/ _ \ /    \ / _ \ __/ _` | |")
  print("  / _  \ (_| | | | | (_| | (_) / /\/\ \  __/ || (_| | |")
  print("  \/ \_/\__,_|_| |_|\__,_|\___/\/    \/\___|\__\__,_|_|")
  print("")
  print("|--------------------------------------------------------|")
  print("    A little script to randomly play a song taken ")
  print("         from 'metal-archives.com' | Zar - 2016")
  print("|--------------------------------------------------------|")
  print(" Options : ")
  print("  -y [--youtube] : Try to find and play song only on Youtube.")
  print("  -n [--nbr]     : Set the number of band to search. One by default.")
  print("  -p [--page]    : Open the archive-metal page of the band.")
  print("  -v [--verbose] : Display more informations.")

def clean_name(name):
  """ Clean the caracters who are not decoded.  """

  name = urllib.parse.unquote(name)
  name = name.replace('_', ' ')
  return name

def get_name_id(html):
  """
  Get the name and the id of the band corresponding to the html page.
  """

  raw_data = BeautifulSoup(html, "html.parser").find('h1', {"class": 'band_name'})
  raw_data = raw_data.find('a').get('href')
  raw_data = str(raw_data).split('/', 5)
  name = raw_data[4]
  ide = raw_data[5]

  return name, ide

def get_cd(ide):
  """
  Take an id and return the name of the newest album/demo/single of a band.
  """

  cd = None
  raw_data  = []
  class_word = ["demo", "single", "album", "other"]
  url_cd = "https://www.metal-archives.com/band/discography/id/" + str(ide) + "/tab/all"

  html = requests.get(url_cd).content
  for i in range(len(class_word)):
    raw_data += BeautifulSoup(html, "html.parser").findAll('a', {"class" :class_word[i]})

  try:
    cd = ''.join(raw_data[(len(raw_data))-1].findAll(text=True))
  except IndexError:
    print("[!] No album for the band\n")

  return cd

def get_related_links(html):
  """
  Take the link in the 'Related links' onglet of archive-metal.
  """

  url = None
  raw_data = BeautifulSoup(html, "html.parser").find('a', {"title": "Related links"})

  if raw_data is not None:
    url = str(raw_data['href'])
  else:
    print("[!] No related link for this band\n")

  return url

def get_music_link(html, name):
  """
  Extract links
  """

  i = 0
  raw_data = []
  music_link = []
  title_list = ["Go to: Myspace", "Go to: Myspace Page", "Go to: Soundcloud", "Go to: Youtube", "Go to: Youtube Page", "Go to: Bandcamp", "Go to: MySpace", "Go to: MySpace Page", "Go to: SoundCloud", "Go to: YouTube", "Go to: YouTube Page", "Go to: YouTube Channel", "Go to: BandCamp", "Go to: "+name+" @ Myspace", "Go to: "+name+" @ Soundcloud", "Go to: "+name+" @ youtube", "Go to: "+name+" @ Bandcamp", "Go to: "+name+" @ MySpace", "Go to: "+name+" @ SoundCloud", "Go to: "+name+" @ youTube", "Go to: "+name+" @ BandCamp"]

  while (raw_data is None) or (i < (len(title_list))):
    raw_data += BeautifulSoup(html, "html.parser").findAll('a', {"title": title_list[i]})
    i += 1

  for j in range(len(raw_data)):
    music_link.append(raw_data[j].get('href'))

  return music_link

def chose_link(list_link):
  """
  Function who return musical link
  """

  site_order = ["bandcamp", "soundcloud", "youtube", "myspace"]

  for site in site_order:
    for link in list_link:
      if site in link:
        return link

  return None

def get_url_youtube(html):
  """
  Take the first video link of a html of Youtube.
  """

  url = None
  raw_data = BeautifulSoup(html, "html.parser").find('a', {"dir": "ltr"})
  if raw_data is not None:
    url = str(raw_data['href'])
  else:
    print("[!] No video clip of the band in Youtube\n")

  return url

def request_youtube(name, cd):
  """"
  Make a request to Youtube and return the first link.
  """

  YT_URL = "https://www.youtube.com/results?search_query="

  if cd is None:
    query = name + "+metal+music"
  else:
    query = name + "+-+" + cd

  if glob_verbose:
    print("[*] Query: " + query.replace('+', ' '))

  url_yt = YT_URL + query
  html_yt = requests.get(url_yt).content
  link_yt = get_url_youtube(html_yt)

  return link_yt

def only_youtube(name, ide, first):
  """
  Function who try to play the song.
  """

  cd = get_cd(ide)
  link = request_youtube(name, cd)

  if link is not None:
    yt_link = "https://www.youtube.com" + link
    webbrowser.open(yt_link)
    return yt_link

def find_band(only_yt):
  """ Find url for one band and display it. """

  html = requests.get(URL_RANDOM).content
  name, ide = get_name_id(html)
  name = clean_name(name)

  if glob_print_page:
    webbrowser.open("https://www.metal-archives.com/bands/" + name + "/" + ide)

  print("[#] Band: " + name + ", ID: " + ide)

  if only_yt:
    return only_youtube(name, ide, True)
  else:
    url = get_related_links(html)
    html = requests.get(url).content
    list_link = get_music_link(html, name)

    link = None
    if list_link:
      link = chose_link(list_link)

    if link:
      webbrowser.open(link)
      return link
    else:
      print("[!] No musical link for this band.\n    Trying on youtube.")
      return only_youtube(name, ide, False)

"""
Main
"""

def main(argv):

  nbr = 1
  only_yt = False
  global glob_verbose
  global glob_print_page

  try:
    opts, args = getopt.getopt(argv, "hyvpn:", ["help", "youtube", "verbose", "page", "nbr"])
  except getopt.GetoptError:
    print("[!] Unhandled option.")
    usage()
    sys.exit(2)

  for opt, arg in opts:
    if opt in ("-h", "--help"):
      usage()
      sys.exit()
    elif opt in ("-y", "--youtube"):
      only_yt = True
    elif opt in ("-v", "--verbose"):
      glob_verbose = True
    elif opt in ("-p", "--page"):
      glob_print_page = True
    elif opt in ("-n", "--nbr"):
      nbr = int(arg)
    else:
      print("[!] Unhandled option.")
      usage()
      sys.exit(2)

  links = []
  for i in range(nbr):
    links.append(find_band(only_yt))
    if glob_verbose:
      print("[*] Link : " + links[i])
    print("    ====")

if __name__ == "__main__":
  main(sys.argv[1:])
<commit_msg>Refactor from getopt -> argparse<commit_after>#!/usr/bin/env python3
# -*- coding: utf8 -*-

import sys
import urllib
import webbrowser
import argparse

from bs4 import BeautifulSoup
import requests

URL_RANDOM = "https://www.metal-archives.com/band/random"
glob_verbose = False
glob_print_page = False

def clean_name(name):
  """ Clean the caracters who are not decoded.  """

  name = urllib.parse.unquote(name)
  name = name.replace('_', ' ')
  return name

def get_name_id(html):
  """
  Get the name and the id of the band corresponding to the html page.
  """

  raw_data = BeautifulSoup(html, "html.parser").find('h1', {"class": 'band_name'})
  raw_data = raw_data.find('a').get('href')
  raw_data = str(raw_data).split('/', 5)
  name = raw_data[4]
  ide = raw_data[5]

  return name, ide

def get_cd(ide):
  """
  Take an id and return the name of the newest album/demo/single of a band.
  """

  cd = None
  raw_data  = []
  class_word = ["demo", "single", "album", "other"]
  url_cd = "https://www.metal-archives.com/band/discography/id/" + str(ide) + "/tab/all"

  html = requests.get(url_cd).content
  for i in range(len(class_word)):
    raw_data += BeautifulSoup(html, "html.parser").findAll('a', {"class" :class_word[i]})

  try:
    cd = ''.join(raw_data[(len(raw_data))-1].findAll(text=True))
  except IndexError:
    print("[!] No album for the band\n")

  return cd

def get_related_links(html):
  """
  Take the link in the 'Related links' onglet of archive-metal.
  """

  url = None
  raw_data = BeautifulSoup(html, "html.parser").find('a', {"title": "Related links"})

  if raw_data is not None:
    url = str(raw_data['href'])
  else:
    print("[!] No related link for this band\n")

  return url

def get_music_link(html, name):
  """
  Extract links
  """

  i = 0
  raw_data = []
  music_link = []
  title_list = ["Go to: Myspace", "Go to: Myspace Page", "Go to: Soundcloud", "Go to: Youtube", "Go to: Youtube Page", "Go to: Bandcamp", "Go to: MySpace", "Go to: MySpace Page", "Go to: SoundCloud", "Go to: YouTube", "Go to: YouTube Page", "Go to: YouTube Channel", "Go to: BandCamp", "Go to: "+name+" @ Myspace", "Go to: "+name+" @ Soundcloud", "Go to: "+name+" @ youtube", "Go to: "+name+" @ Bandcamp", "Go to: "+name+" @ MySpace", "Go to: "+name+" @ SoundCloud", "Go to: "+name+" @ youTube", "Go to: "+name+" @ BandCamp"]

  while (raw_data is None) or (i < (len(title_list))):
    raw_data += BeautifulSoup(html, "html.parser").findAll('a', {"title": title_list[i]})
    i += 1

  for j in range(len(raw_data)):
    music_link.append(raw_data[j].get('href'))

  return music_link

def chose_link(list_link):
  """
  Function who return musical link
  """

  site_order = ["bandcamp", "soundcloud", "youtube", "myspace"]

  for site in site_order:
    for link in list_link:
      if site in link:
        return link

  return None

def get_url_youtube(html):
  """
  Take the first video link of a html of Youtube.
  """

  url = None
  raw_data = BeautifulSoup(html, "html.parser").find('a', {"dir": "ltr"})
  if raw_data is not None:
    url = str(raw_data['href'])
  else:
    print("[!] No video clip of the band in Youtube\n")

  return url

def request_youtube(name, cd):
  """"
  Make a request to Youtube and return the first link.
  """

  YT_URL = "https://www.youtube.com/results?search_query="

  if cd is None:
    query = name + "+metal+music"
  else:
    query = name + "+-+" + cd

  if glob_verbose:
    print("[*] Query: " + query.replace('+', ' '))

  url_yt = YT_URL + query
  html_yt = requests.get(url_yt).content
  link_yt = get_url_youtube(html_yt)

  return link_yt

def only_youtube(name, ide, first):
  """
  Function who try to play the song.
  """

  cd = get_cd(ide)
  link = request_youtube(name, cd)

  if link is not None:
    yt_link = "https://www.youtube.com" + link
    webbrowser.open(yt_link)
    return yt_link

def find_band(only_yt):
  """ Find url for one band and display it. """

  html = requests.get(URL_RANDOM).content
  name, ide = get_name_id(html)
  name = clean_name(name)

  if glob_print_page:
    webbrowser.open("https://www.metal-archives.com/bands/" + name + "/" + ide)

  print("[#] Band: " + name + ", ID: " + ide)

  if only_yt:
    return only_youtube(name, ide, True)
  else:
    url = get_related_links(html)
    html = requests.get(url).content
    list_link = get_music_link(html, name)

    link = None
    if list_link:
      link = chose_link(list_link)

    if link:
      webbrowser.open(link)
      return link
    else:
      print("[!] No musical link for this band.\n    Trying on youtube.")
      return only_youtube(name, ide, False)

"""
Main
"""

def main(argv):
  parser = argparse.ArgumentParser(
    prog='randometal',
    description='Randomly select bands from "metal-archives.com".',
    usage="""
       __                 _                   _        _
      /__\ __ _ _ __   __| | ___   /\/\   ___| |_ __ _| |
     / \/// _` | '_ \ / _` |/ _ \ /    \ / _ \ __/ _` | |
    / _  \ (_| | | | | (_| | (_) / /\/\ \  __/ || (_| | |
    \/ \_/\__,_|_| |_|\__,_|\___/\/    \/\___|\__\__,_|_|
  """)
  parser.add_argument('-y', '--youtube', action='store_true', help='try to find an play songs only on YouTube')
  parser.add_argument('-v', '--verbose', action='store_true', help='display more informations')
  parser.add_argument('-p', '--page', action='store_true', help='open the archive-metal page of the band')
  parser.add_argument('-n', '--nbr', default=1, type=int, help='number of bands to search. One by default')

  args = parser.parse_args()

  global glob_verbose
  glob_verbose = args.verbose
  global glob_print_page
  glob_print_page = args.page

  links = []
  for i in range(args.nbr):
    links.append(find_band(args.youtube))
    if args.verbose:
      print("[*] Link : " + links[i] + "\n    ====")

if __name__ == "__main__":
  main(sys.argv[1:])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf8 -*-

import sys
import urllib
import webbrowser
import argparse

from bs4 import BeautifulSoup
import requests

URL_RANDOM = "https://www.metal-archives.com/band/random"
glob_verbose = False
glob_print_page = False

def clean_name(name):
  """ Clean the caracters who are not decoded.  """

  name = urllib.parse.unquote(name)
  name = name.replace('_', ' ')
  return name

def get_name_id(html):
  """
  Get the name and the id of the band corresponding to the html page.
  """

  raw_data = BeautifulSoup(html, "html.parser").find('h1', {"class": 'band_name'})
  raw_data = raw_data.find('a').get('href')
  raw_data = str(raw_data).split('/', 5)
  name = raw_data[4]
  ide = raw_data[5]

  return name, ide

def get_cd(ide):
  """
  Take an id and return the name of the newest album/demo/single of a band.
  """

  cd = None
  raw_data  = []
  class_word = ["demo", "single", "album", "other"]
  url_cd = "https://www.metal-archives.com/band/discography/id/" + str(ide) + "/tab/all"

  html = requests.get(url_cd).content
  for i in range(len(class_word)):
    raw_data += BeautifulSoup(html, "html.parser").findAll('a', {"class" :class_word[i]})

  try:
    cd = ''.join(raw_data[(len(raw_data))-1].findAll(text=True))
  except IndexError:
    print("[!] No album for the band\n")

  return cd

def get_related_links(html):
  """
  Take the link in the 'Related links' onglet of archive-metal.
  """

  url = None
  raw_data = BeautifulSoup(html, "html.parser").find('a', {"title": "Related links"})

  if raw_data is not None:
    url = str(raw_data['href'])
  else:
    print("[!] No related link for this band\n")

  return url

def get_music_link(html, name):
  """
  Extract links
  """

  i = 0
  raw_data = []
  music_link = []
  title_list = ["Go to: Myspace", "Go to: Myspace Page", "Go to: Soundcloud", "Go to: Youtube", "Go to: Youtube Page", "Go to: Bandcamp", "Go to: MySpace", "Go to: MySpace Page", "Go to: SoundCloud", "Go to: YouTube", "Go to: YouTube Page", "Go to: YouTube Channel", "Go to: BandCamp", "Go to: "+name+" @ Myspace", "Go to: "+name+" @ Soundcloud", "Go to: "+name+" @ youtube", "Go to: "+name+" @ Bandcamp", "Go to: "+name+" @ MySpace", "Go to: "+name+" @ SoundCloud", "Go to: "+name+" @ youTube", "Go to: "+name+" @ BandCamp"]

  while (raw_data is None) or (i < (len(title_list))):
    raw_data += BeautifulSoup(html, "html.parser").findAll('a', {"title": title_list[i]})
    i += 1

  for j in range(len(raw_data)):
    music_link.append(raw_data[j].get('href'))

  return music_link

def chose_link(list_link):
  """
  Function who return musical link
  """

  site_order = ["bandcamp", "soundcloud", "youtube", "myspace"]

  for site in site_order:
    for link in list_link:
      if site in link:
        return link

  return None

def get_url_youtube(html):
  """
  Take the first video link of a html of Youtube.
  """

  url = None
  raw_data = BeautifulSoup(html, "html.parser").find('a', {"dir": "ltr"})
  if raw_data is not None:
    url = str(raw_data['href'])
  else:
    print("[!] No video clip of the band in Youtube\n")

  return url

def request_youtube(name, cd):
  """"
  Make a request to Youtube and return the first link.
  """

  YT_URL = "https://www.youtube.com/results?search_query="

  if cd is None:
    query = name + "+metal+music"
  else:
    query = name + "+-+" + cd

  if glob_verbose:
    print("[*] Query: " + query.replace('+', ' '))

  url_yt = YT_URL + query
  html_yt = requests.get(url_yt).content
  link_yt = get_url_youtube(html_yt)

  return link_yt

def only_youtube(name, ide, first):
  """
  Function who try to play the song.
  """

  cd = get_cd(ide)
  link = request_youtube(name, cd)

  if link is not None:
    yt_link = "https://www.youtube.com" + link
    webbrowser.open(yt_link)
    return yt_link

def find_band(only_yt):
  """ Find url for one band and display it. """

  html = requests.get(URL_RANDOM).content
  name, ide = get_name_id(html)
  name = clean_name(name)

  if glob_print_page:
    webbrowser.open("https://www.metal-archives.com/bands/" + name + "/" + ide)

  print("[#] Band: " + name + ", ID: " + ide)

  if only_yt:
    return only_youtube(name, ide, True)
  else:
    url = get_related_links(html)
    html = requests.get(url).content
    list_link = get_music_link(html, name)

    link = None
    if list_link:
      link = chose_link(list_link)

    if link:
      webbrowser.open(link)
      return link
    else:
      print("[!] No musical link for this band.\n    Trying on youtube.")
      return only_youtube(name, ide, False)

"""
Main
"""

def main(argv):
  parser = argparse.ArgumentParser(
    prog='randometal',
    description='Randomly select bands from "metal-archives.com".',
    usage="""
       __                 _                   _        _
      /__\ __ _ _ __   __| | ___   /\/\   ___| |_ __ _| |
     / \/// _` | '_ \ / _` |/ _ \ /    \ / _ \ __/ _` | |
    / _  \ (_| | | | | (_| | (_) / /\/\ \  __/ || (_| | |
    \/ \_/\__,_|_| |_|\__,_|\___/\/    \/\___|\__\__,_|_|
  """)
  parser.add_argument('-y', '--youtube', action='store_true', help='try to find an play songs only on YouTube')
  parser.add_argument('-v', '--verbose', action='store_true', help='display more informations')
  parser.add_argument('-p', '--page', action='store_true', help='open the archive-metal page of the band')
  parser.add_argument('-n', '--nbr', default=1, type=int, help='number of bands to search. One by default')

  args = parser.parse_args()

  global glob_verbose
  glob_verbose = args.verbose
  global glob_print_page
  glob_print_page = args.page

  links = []
  for i in range(args.nbr):
    links.append(find_band(args.youtube))
    if args.verbose:
      print("[*] Link : " + links[i] + "\n    ====")

if __name__ == "__main__":
  main(sys.argv[1:])
<commit_msg>Print errors to stderr<commit_after>#!/usr/bin/env python3
# -*- coding: utf8 -*-

import sys
import urllib
import webbrowser
import argparse

from bs4 import BeautifulSoup
import requests

URL_RANDOM = "https://www.metal-archives.com/band/random"
glob_verbose = False
glob_print_page = False

def clean_name(name):
  """ Clean the caracters who are not decoded.  """

  name = urllib.parse.unquote(name)
  name = name.replace('_', ' ')
  return name

def get_name_id(html):
  """
  Get the name and the id of the band corresponding to the html page.
  """

  raw_data = BeautifulSoup(html, "html.parser").find('h1', {"class": 'band_name'})
  raw_data = raw_data.find('a').get('href')
  raw_data = str(raw_data).split('/', 5)
  name = raw_data[4]
  ide = raw_data[5]

  return name, ide

def get_cd(ide):
  """
  Take an id and return the name of the newest album/demo/single of a band.
  """

  cd = None
  raw_data  = []
  class_word = ["demo", "single", "album", "other"]
  url_cd = "https://www.metal-archives.com/band/discography/id/" + str(ide) + "/tab/all"

  html = requests.get(url_cd).content
  for i in range(len(class_word)):
    raw_data += BeautifulSoup(html, "html.parser").findAll('a', {"class" :class_word[i]})

  try:
    cd = ''.join(raw_data[(len(raw_data))-1].findAll(text=True))
  except IndexError:
    print("[!] No album for the band.", file=sys.stderr)

  return cd

def get_related_links(html):
  """
  Take the link in the 'Related links' onglet of archive-metal.
  """

  url = None
  raw_data = BeautifulSoup(html, "html.parser").find('a', {"title": "Related links"})

  if raw_data is not None:
    url = str(raw_data['href'])
  else:
    print("[!] No related link for this band.", file=sys.stderr)

  return url

def get_music_link(html, name):
  """
  Extract links
  """

  i = 0
  raw_data = []
  music_link = []
  title_list = ["Go to: Myspace", "Go to: Myspace Page", "Go to: Soundcloud", "Go to: Youtube", "Go to: Youtube Page", "Go to: Bandcamp", "Go to: MySpace", "Go to: MySpace Page", "Go to: SoundCloud", "Go to: YouTube", "Go to: YouTube Page", "Go to: YouTube Channel", "Go to: BandCamp", "Go to: "+name+" @ Myspace", "Go to: "+name+" @ Soundcloud", "Go to: "+name+" @ youtube", "Go to: "+name+" @ Bandcamp", "Go to: "+name+" @ MySpace", "Go to: "+name+" @ SoundCloud", "Go to: "+name+" @ youTube", "Go to: "+name+" @ BandCamp"]

  while (raw_data is None) or (i < (len(title_list))):
    raw_data += BeautifulSoup(html, "html.parser").findAll('a', {"title": title_list[i]})
    i += 1

  for j in range(len(raw_data)):
    music_link.append(raw_data[j].get('href'))

  return music_link

def chose_link(list_link):
  """
  Function who return musical link
  """

  site_order = ["bandcamp", "soundcloud", "youtube", "myspace"]

  for site in site_order:
    for link in list_link:
      if site in link:
        return link

  return None

def get_url_youtube(html):
  """
  Take the first video link of a html of Youtube.
  """

  url = None
  raw_data = BeautifulSoup(html, "html.parser").find('a', {"dir": "ltr"})
  if raw_data is not None:
    url = str(raw_data['href'])
  else:
    print("[!] No video clip of the band in Youtube.", file=sys.stderr)

  return url

def request_youtube(name, cd):
  """"
  Make a request to Youtube and return the first link.
  """

  YT_URL = "https://www.youtube.com/results?search_query="

  if cd is None:
    query = name + "+metal+music"
  else:
    query = name + "+-+" + cd

  if glob_verbose:
    print("[*] Query: " + query.replace('+', ' '))

  url_yt = YT_URL + query
  html_yt = requests.get(url_yt).content
  link_yt = get_url_youtube(html_yt)

  return link_yt

def only_youtube(name, ide, first):
  """
  Function who try to play the song.
  """

  cd = get_cd(ide)
  link = request_youtube(name, cd)

  if link is not None:
    yt_link = "https://www.youtube.com" + link
    webbrowser.open(yt_link)
    return yt_link

def find_band(only_yt):
  """ Find url for one band and display it. """

  html = requests.get(URL_RANDOM).content
  name, ide = get_name_id(html)
  name = clean_name(name)

  if glob_print_page:
    webbrowser.open("https://www.metal-archives.com/bands/" + name + "/" + ide)

  print("[#] Band: " + name + ", ID: " + ide)

  if only_yt:
    return only_youtube(name, ide, True)
  else:
    url = get_related_links(html)
    html = requests.get(url).content
    list_link = get_music_link(html, name)

    link = None
    if list_link:
      link = chose_link(list_link)

    if link:
      webbrowser.open(link)
      return link
    else:
      print("[!] No musical link for this band.\n    Trying on youtube.", file=sys.stderr)
      return only_youtube(name, ide, False)

"""
Main
"""

def main(argv):
  parser = argparse.ArgumentParser(
    prog='randometal',
    description='Randomly select bands from "metal-archives.com".',
    usage="""
       __                 _                   _        _
      /__\ __ _ _ __   __| | ___   /\/\   ___| |_ __ _| |
     / \/// _` | '_ \ / _` |/ _ \ /    \ / _ \ __/ _` | |
    / _  \ (_| | | | | (_| | (_) / /\/\ \  __/ || (_| | |
    \/ \_/\__,_|_| |_|\__,_|\___/\/    \/\___|\__\__,_|_|
  """)
  parser.add_argument('-y', '--youtube', action='store_true', help='try to find an play songs only on YouTube')
  parser.add_argument('-v', '--verbose', action='store_true', help='display more informations')
  parser.add_argument('-p', '--page', action='store_true', help='open the archive-metal page of the band')
  parser.add_argument('-n', '--nbr', default=1, type=int, help='number of bands to search. One by default')

  args = parser.parse_args()

  global glob_verbose
  glob_verbose = args.verbose
  global glob_print_page
  glob_print_page = args.page

  links = []
  for i in range(args.nbr):
    links.append(find_band(args.youtube))
    if args.verbose:
      print("[*] Link : " + links[i] + "\n    ====")

if __name__ == "__main__":
  main(sys.argv[1:])
<|endoftext|>


<commit_before># --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

import argparse
import json
from docutils import nodes
from docutils.statemachine import ViewList
from sphinx.util.compat import Directive
from sphinx.util.nodes import nested_parse_with_titles

from azure.cli.core.application import Application, Configuration
import azure.cli.core._help as _help

app = Application(Configuration())
for cmd in app.configuration.get_command_table():
    try:
        app.execute(cmd.split() + ['-h'])
    except:
        pass

class AzHelpGenDirective(Directive):
    def make_rst(self):
        INDENT = '   '
        DOUBLEINDENT = INDENT * 2
        parser_keys = []
        parser_values = []
        _store_parsers(app.parser, parser_keys, parser_values)
        doc_source_map = _load_doc_source_map()

        help_files = []
        for cmd, parser in zip(parser_keys, parser_values):
            try:
                help_file = _help.GroupHelpFile(cmd, parser) if _is_group(parser) else _help.CommandHelpFile(cmd, parser)
                help_file.load(parser)
                help_files.append(help_file)
            except Exception as ex:
                print("Skipped '{}' due to '{}'".format(cmd, ex))
        help_files = sorted(help_files, key=lambda x: x.command)

        for help_file in help_files:
            is_command = isinstance(help_file, _help.CommandHelpFile)
            yield '.. cli{}:: {}'.format('command' if is_command else 'group', help_file.command if help_file.command else 'az') #it is top level group az if command is empty
            yield ''
            yield '{}:summary: {}'.format(INDENT, help_file.short_summary)
            yield '{}:description: {}'.format(INDENT, help_file.long_summary)
            if not is_command:
                top_group_name = help_file.command.split()[0] if help_file.command else 'az'
                yield '{}:docsource: {}'.format(INDENT, doc_source_map[top_group_name] if top_group_name in doc_source_map else '')
            else:
                top_command_name = help_file.command.split()[0] if help_file.command else ''
                if top_command_name in doc_source_map:
                    yield '{}:docsource: {}'.format(INDENT, doc_source_map[top_command_name])
            yield ''

            if is_command and help_file.parameters:
               group_registry = _help.ArgumentGroupRegistry(
                  [p.group_name for p in help_file.parameters if p.group_name])

               for arg in sorted(help_file.parameters,
                                key=lambda p: group_registry.get_group_priority(p.group_name)
                                + str(not p.required) + p.name):
                  yield '{}.. cliarg:: {}'.format(INDENT, arg.name)
                  yield ''
                  yield '{}:required: {}'.format(DOUBLEINDENT, arg.required)
                  short_summary = arg.short_summary or ''
                  possible_values_index = short_summary.find(' Possible values include')
                  short_summary = short_summary[0:possible_values_index
                                                if possible_values_index >= 0 else len(short_summary)]
                  short_summary = short_summary.strip()
                  yield '{}:summary: {}'.format(DOUBLEINDENT, short_summary)
                  yield '{}:description: {}'.format(DOUBLEINDENT, arg.long_summary)
                  if arg.choices:
                     yield '{}:values: {}'.format(DOUBLEINDENT, ', '.join(sorted([str(x) for x in arg.choices])))
                  if arg.default and arg.default != argparse.SUPPRESS:
                     yield '{}:default: {}'.format(DOUBLEINDENT, arg.default)
                  if arg.value_sources:
                     yield '{}:source: {}'.format(DOUBLEINDENT, ', '.join(arg.value_sources))
                  yield ''
            yield ''
            if len(help_file.examples) > 0:
               for e in help_file.examples:
                  yield '{}.. cliexample:: {}'.format(INDENT, e.name)
                  yield ''
                  yield DOUBLEINDENT + e.text
                  yield ''

    def run(self):
        node = nodes.section()
        node.document = self.state.document
        result = ViewList()
        for line in self.make_rst():
            result.append(line, '<azhelpgen>')

        nested_parse_with_titles(self.state, result, node)
        return node.children

def setup(app):
    app.add_directive('azhelpgen', AzHelpGenDirective)

def _store_parsers(parser, parser_keys, parser_values):
    for s in parser.subparsers.values():
        parser_keys.append(_get_parser_name(s))
        parser_values.append(s)
        if _is_group(s):
            for c in s.choices.values():
                parser_keys.append(_get_parser_name(c))
                parser_values.append(c)
                _store_parsers(c, parser_keys, parser_values)

def _load_doc_source_map():
    with open('azhelpgen/doc_source_map.json') as open_file:
        return json.load(open_file)

def _is_group(parser):
    return getattr(parser, '_subparsers', None) is not None \
        or getattr(parser, 'choices', None) is not None

def _get_parser_name(s):
    return (s._prog_prefix if hasattr(s, '_prog_prefix') else s.prog)[3:]
<commit_msg>[Docs CI] Fix the command group duplication error (#4219)<commit_after># --------------------------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

import argparse
import json
from docutils import nodes
from docutils.statemachine import ViewList
from sphinx.util.compat import Directive
from sphinx.util.nodes import nested_parse_with_titles

from azure.cli.core.application import Application, Configuration
import azure.cli.core._help as _help

app = Application(Configuration())
for cmd in app.configuration.get_command_table():
    try:
        app.execute(cmd.split() + ['-h'])
    except:
        pass

class AzHelpGenDirective(Directive):
    def make_rst(self):
        INDENT = '   '
        DOUBLEINDENT = INDENT * 2
        parser_keys = []
        parser_values = []
        sub_parser_keys = []
        sub_parser_values = []
        _store_parsers(app.parser, parser_keys, parser_values, sub_parser_keys, sub_parser_values)
        for cmd, parser in zip(parser_keys, parser_values):
            if cmd not in sub_parser_keys:
                sub_parser_keys.append(cmd)
                sub_parser_values.append(parser)
        doc_source_map = _load_doc_source_map()
        help_files = []
        for cmd, parser in zip(sub_parser_keys, sub_parser_values):
            try:
                help_file = _help.GroupHelpFile(cmd, parser) if _is_group(parser) else _help.CommandHelpFile(cmd, parser)
                help_file.load(parser)
                help_files.append(help_file)
            except Exception as ex:
                print("Skipped '{}' due to '{}'".format(cmd, ex))
        help_files = sorted(help_files, key=lambda x: x.command)

        for help_file in help_files:
            is_command = isinstance(help_file, _help.CommandHelpFile)
            yield '.. cli{}:: {}'.format('command' if is_command else 'group', help_file.command if help_file.command else 'az') #it is top level group az if command is empty
            yield ''
            yield '{}:summary: {}'.format(INDENT, help_file.short_summary)
            yield '{}:description: {}'.format(INDENT, help_file.long_summary)
            if not is_command:
                top_group_name = help_file.command.split()[0] if help_file.command else 'az'
                yield '{}:docsource: {}'.format(INDENT, doc_source_map[top_group_name] if top_group_name in doc_source_map else '')
            else:
                top_command_name = help_file.command.split()[0] if help_file.command else ''
                if top_command_name in doc_source_map:
                    yield '{}:docsource: {}'.format(INDENT, doc_source_map[top_command_name])
            yield ''

            if is_command and help_file.parameters:
               group_registry = _help.ArgumentGroupRegistry(
                  [p.group_name for p in help_file.parameters if p.group_name])

               for arg in sorted(help_file.parameters,
                                key=lambda p: group_registry.get_group_priority(p.group_name)
                                + str(not p.required) + p.name):
                  yield '{}.. cliarg:: {}'.format(INDENT, arg.name)
                  yield ''
                  yield '{}:required: {}'.format(DOUBLEINDENT, arg.required)
                  short_summary = arg.short_summary or ''
                  possible_values_index = short_summary.find(' Possible values include')
                  short_summary = short_summary[0:possible_values_index
                                                if possible_values_index >= 0 else len(short_summary)]
                  short_summary = short_summary.strip()
                  yield '{}:summary: {}'.format(DOUBLEINDENT, short_summary)
                  yield '{}:description: {}'.format(DOUBLEINDENT, arg.long_summary)
                  if arg.choices:
                     yield '{}:values: {}'.format(DOUBLEINDENT, ', '.join(sorted([str(x) for x in arg.choices])))
                  if arg.default and arg.default != argparse.SUPPRESS:
                     yield '{}:default: {}'.format(DOUBLEINDENT, arg.default)
                  if arg.value_sources:
                     yield '{}:source: {}'.format(DOUBLEINDENT, ', '.join(arg.value_sources))
                  yield ''
            yield ''
            if len(help_file.examples) > 0:
               for e in help_file.examples:
                  yield '{}.. cliexample:: {}'.format(INDENT, e.name)
                  yield ''
                  yield DOUBLEINDENT + e.text
                  yield ''

    def run(self):
        node = nodes.section()
        node.document = self.state.document
        result = ViewList()
        for line in self.make_rst():
            result.append(line, '<azhelpgen>')

        nested_parse_with_titles(self.state, result, node)
        return node.children

def setup(app):
    app.add_directive('azhelpgen', AzHelpGenDirective)

def _store_parsers(parser, parser_keys, parser_values, sub_parser_keys, sub_parser_values):
    for s in parser.subparsers.values():
        parser_keys.append(_get_parser_name(s))
        parser_values.append(s)
        if _is_group(s):
            for c in s.choices.values():
                sub_parser_keys.append(_get_parser_name(c))
                sub_parser_values.append(c)
                _store_parsers(c, parser_keys, parser_values, sub_parser_keys, sub_parser_values)

def _load_doc_source_map():
    with open('azhelpgen/doc_source_map.json') as open_file:
        return json.load(open_file)

def _is_group(parser):
    return getattr(parser, '_subparsers', None) is not None \
        or getattr(parser, 'choices', None) is not None

def _get_parser_name(s):
    return (s._prog_prefix if hasattr(s, '_prog_prefix') else s.prog)[3:]
<|endoftext|>


<commit_before>import unittest
from copy import deepcopy
import allegedb

testkvs = [0, 1, 10, 10**10, 10**10**4, 'spam', 'eggs', 'ham',  '💧', '🔑', '𐦖',('spam', 'eggs', 'ham')]
testvs = [['spam', 'eggs', 'ham'], {'foo': 'bar', 0: 1, '💧': '🔑'}]
testdata = []
for k in testkvs:
    for v in testkvs:
        testdata.append((k, v))
    for v in testvs:
        testdata.append((k, v))
testdata.append(('lol', deepcopy(testdata)))

class AllegedTest(unittest.TestCase):
    def setUp(self):
        self.engine = allegedb.ORM('sqlite:///:memory:')
        self.engine.initdb()
        self.graphmakers = (self.engine.new_graph, self.engine.new_digraph, self.engine.new_multigraph, self.engine.new_multidigraph)

    def tearDown(self):
        self.engine.close()

class GraphTest(AllegedTest):
    def setUp(self):
        super().setUp()
        g = self.engine.new_graph('test')
        g.add_node(0)
        self.assertIn(0, g)
        g.add_node(1)
        self.assertIn(1, g)
        g.add_edge(0, 1)
        self.assertIn(1, g.adj[0])
        self.assertIn(0, g.adj[1])
        # TODO: test adding edges whose nodes do not yet exist
        self.engine.rev = 1
        self.assertIn(0, g)
        self.assertIn(1, g)
        self.engine.branch = 'no_edge'
        self.assertIn(0, g)
        self.assertIn(1, g)
        self.assertIn(1, g.adj[0])
        self.assertIn(0, g.adj[1])
        g.remove_edge(0, 1)
        self.assertIn(0, g)
        self.assertIn(1, g)
        self.assertNotIn(0, g.adj[1])
        self.assertNotIn(1, g.adj[0])
        self.engine.branch = 'triangle'
        g.add_node(2)
        self.assertIn(2, g)
        g.add_edge(0, 1)
        g.add_edge(1, 2)
        g.add_edge(2, 0)
        self.engine.branch = 'square'
        self.engine.rev = 2
        self.assertIn(2, g)
        self.assertIn(2, list(g.node.keys()))
        g.remove_edge(2, 0)
        g.add_node(3)
        g.add_edge(2, 3)
        g.add_edge(3, 0)
        self.engine.branch = 'nothing'
        g.remove_nodes_from((0, 1, 2, 3))
        self.engine.branch = 'trunk'
        self.engine.rev = 0

class BranchLineageTest(GraphTest):
    def runTest(self):
        """Create some branches of history and check that allegedb remembers where
        each came from and what happened in each.

        """
        self.assertTrue(self.engine.is_parent_of('trunk', 'no_edge'))
        self.assertTrue(self.engine.is_parent_of('trunk', 'triangle'))
        self.assertTrue(self.engine.is_parent_of('trunk', 'nothing'))
        self.assertTrue(self.engine.is_parent_of('no_edge', 'triangle'))
        self.assertTrue(self.engine.is_parent_of('square', 'nothing'))
        self.assertFalse(self.engine.is_parent_of('nothing', 'trunk'))
        self.assertFalse(self.engine.is_parent_of('triangle', 'no_edge'))
        g = self.engine.graph['test']
        self.assertIn(0, g.node)
        self.assertIn(1, g.node)
        self.assertIn(0, g.edge)
        self.assertIn(1, g.edge[0])
        self.engine.rev = 0

        def badjump():
            self.engine.branch = 'no_edge'
        self.assertRaises(ValueError, badjump)
        self.engine.rev = 2
        self.engine.branch = 'no_edge'
        self.assertIn(0, g)
        self.assertIn(0, list(g.node.keys()))
        self.assertNotIn(1, g.edge[0])
        self.assertRaises(KeyError, lambda: g.edge[0][1])
        self.engine.branch = 'triangle'
        self.assertIn(2, g.node)
        for orig in (0, 1, 2):
            for dest in (0, 1, 2):
                if orig == dest:
                    continue
                self.assertIn(orig, g.edge)
                self.assertIn(dest, g.edge[orig])
        self.engine.branch = 'square'
        self.assertNotIn(0, g.edge[2])
        self.assertRaises(KeyError, lambda: g.edge[2][0])
        self.engine.rev = 2
        self.assertIn(3, g.node)
        self.assertIn(1, g.edge[0])
        self.assertIn(2, g.edge[1])
        self.assertIn(3, g.edge[2])
        self.assertIn(0, g.edge[3])
        self.engine.branch = 'nothing'
        for node in (0, 1, 2):
            self.assertNotIn(node, g.node)
            self.assertNotIn(node, g.edge)
        self.engine.branch = 'trunk'
        self.engine.rev = 0
        self.assertIn(0, g.node)
        self.assertIn(1, g.node)
        self.assertIn(0, g.edge)
        self.assertIn(1, g.edge[0])

class StorageTest(AllegedTest):
    def runTest(self):
        """Test that all the graph types can store and retrieve key-value pairs
        for the graph as a whole, for nodes, and for edges.

        """
        for graphmaker in self.graphmakers:
            g = graphmaker('testgraph')
            g.add_node(0)
            g.add_node(1)
            g.add_edge(0, 1)
            n = g.node[0]
            e = g.edge[0][1]
            if isinstance(e, allegedb.graph.MultiEdges):
                e = e[0]
            for (k, v) in testdata:
                g.graph[k] = v
                self.assertIn(k, g.graph)
                self.assertEqual(g.graph[k], v)
                del g.graph[k]
                self.assertNotIn(k, g.graph)
                n[k] = v
                self.assertIn(k, n)
                self.assertEqual(n[k], v)
                del n[k]
                self.assertNotIn(k, n)
                e[k] = v
                self.assertIn(k, e)
                self.assertEqual(e[k], v)
                del e[k]
                self.assertNotIn(k, e)
            self.engine.del_graph('testgraph')

class CompiledQueriesTest(AllegedTest):
    def runTest(self):
        """Make sure that the queries generated in SQLAlchemy are the same as
        those precompiled into SQLite.

        """
        from allegedb.alchemy import Alchemist
        self.assertTrue(hasattr(self.engine.query, 'alchemist'))
        self.assertTrue(isinstance(self.engine.query.alchemist, Alchemist))
        from json import load
        with open(self.engine.query.json_path + '/sqlite.json', 'r') as jsonfile:
            precompiled = load(jsonfile)
        self.assertEqual(
            precompiled.keys(), self.engine.query.alchemist.sql.keys()
        )
        for (k, query) in precompiled.items():
            self.assertEqual(
                query,
                str(
                    self.engine.query.alchemist.sql[k]
                )
            )

if __name__ == '__main__':
    unittest.main()
<commit_msg>Amend allegedb tests for changed API<commit_after>import unittest
from copy import deepcopy
import allegedb

testkvs = [0, 1, 10, 10**10, 10**10**4, 'spam', 'eggs', 'ham',  '💧', '🔑', '𐦖',('spam', 'eggs', 'ham')]
testvs = [['spam', 'eggs', 'ham'], {'foo': 'bar', 0: 1, '💧': '🔑'}]
testdata = []
for k in testkvs:
    for v in testkvs:
        testdata.append((k, v))
    for v in testvs:
        testdata.append((k, v))
testdata.append(('lol', deepcopy(testdata)))

class AllegedTest(unittest.TestCase):
    def setUp(self):
        self.engine = allegedb.ORM('sqlite:///:memory:')
        self.graphmakers = (self.engine.new_graph, self.engine.new_digraph, self.engine.new_multigraph, self.engine.new_multidigraph)

    def tearDown(self):
        self.engine.close()

class GraphTest(AllegedTest):
    def setUp(self):
        super().setUp()
        g = self.engine.new_graph('test')
        g.add_node(0)
        self.assertIn(0, g)
        g.add_node(1)
        self.assertIn(1, g)
        g.add_edge(0, 1)
        self.assertIn(1, g.adj[0])
        self.assertIn(0, g.adj[1])
        # TODO: test adding edges whose nodes do not yet exist
        self.engine.turn = 1
        self.assertIn(0, g)
        self.assertIn(1, g)
        self.engine.branch = 'no_edge'
        self.assertIn(0, g)
        self.assertIn(1, g)
        self.assertIn(1, g.adj[0])
        self.assertIn(0, g.adj[1])
        g.remove_edge(0, 1)
        self.assertIn(0, g)
        self.assertIn(1, g)
        self.assertNotIn(0, g.adj[1])
        self.assertNotIn(1, g.adj[0])
        self.engine.branch = 'triangle'
        g.add_node(2)
        self.assertIn(2, g)
        g.add_edge(0, 1)
        g.add_edge(1, 2)
        g.add_edge(2, 0)
        self.engine.branch = 'square'
        self.engine.turn = 2
        self.assertIn(2, g)
        self.assertIn(2, list(g.node.keys()))
        g.remove_edge(2, 0)
        g.add_node(3)
        g.add_edge(2, 3)
        g.add_edge(3, 0)
        self.engine.branch = 'nothing'
        g.remove_nodes_from((0, 1, 2, 3))
        self.engine.branch = 'trunk'
        self.engine.turn = 0

class BranchLineageTest(GraphTest):
    def runTest(self):
        """Create some branches of history and check that allegedb remembers where
        each came from and what happened in each.

        """
        self.assertTrue(self.engine.is_parent_of('trunk', 'no_edge'))
        self.assertTrue(self.engine.is_parent_of('trunk', 'triangle'))
        self.assertTrue(self.engine.is_parent_of('trunk', 'nothing'))
        self.assertTrue(self.engine.is_parent_of('no_edge', 'triangle'))
        self.assertTrue(self.engine.is_parent_of('square', 'nothing'))
        self.assertFalse(self.engine.is_parent_of('nothing', 'trunk'))
        self.assertFalse(self.engine.is_parent_of('triangle', 'no_edge'))
        g = self.engine.graph['test']
        self.assertIn(0, g.node)
        self.assertIn(1, g.node)
        self.assertIn(0, g.edge)
        self.assertIn(1, g.edge[0])
        self.engine.turn = 0

        def badjump():
            self.engine.branch = 'no_edge'
        self.assertRaises(ValueError, badjump)
        self.engine.turn = 2
        self.engine.branch = 'no_edge'
        self.assertIn(0, g)
        self.assertIn(0, list(g.node.keys()))
        self.assertNotIn(1, g.edge[0])
        self.assertRaises(KeyError, lambda: g.edge[0][1])
        self.engine.branch = 'triangle'
        self.assertIn(2, g.node)
        for orig in (0, 1, 2):
            for dest in (0, 1, 2):
                if orig == dest:
                    continue
                self.assertIn(orig, g.edge)
                self.assertIn(dest, g.edge[orig])
        self.engine.branch = 'square'
        self.assertNotIn(0, g.edge[2])
        self.assertRaises(KeyError, lambda: g.edge[2][0])
        self.engine.turn = 2
        self.assertIn(3, g.node)
        self.assertIn(1, g.edge[0])
        self.assertIn(2, g.edge[1])
        self.assertIn(3, g.edge[2])
        self.assertIn(0, g.edge[3])
        self.engine.branch = 'nothing'
        for node in (0, 1, 2):
            self.assertNotIn(node, g.node)
            self.assertNotIn(node, g.edge)
        self.engine.branch = 'trunk'
        self.engine.turn = 0
        self.assertIn(0, g.node)
        self.assertIn(1, g.node)
        self.assertIn(0, g.edge)
        self.assertIn(1, g.edge[0])

class StorageTest(AllegedTest):
    def runTest(self):
        """Test that all the graph types can store and retrieve key-value pairs
        for the graph as a whole, for nodes, and for edges.

        """
        for graphmaker in self.graphmakers:
            g = graphmaker('testgraph')
            g.add_node(0)
            g.add_node(1)
            g.add_edge(0, 1)
            n = g.node[0]
            e = g.edge[0][1]
            if isinstance(e, allegedb.graph.MultiEdges):
                e = e[0]
            for (k, v) in testdata:
                g.graph[k] = v
                self.assertIn(k, g.graph)
                self.assertEqual(g.graph[k], v)
                del g.graph[k]
                self.assertNotIn(k, g.graph)
                n[k] = v
                self.assertIn(k, n)
                self.assertEqual(n[k], v)
                del n[k]
                self.assertNotIn(k, n)
                e[k] = v
                self.assertIn(k, e)
                self.assertEqual(e[k], v)
                del e[k]
                self.assertNotIn(k, e)
            self.engine.del_graph('testgraph')

class CompiledQueriesTest(AllegedTest):
    def runTest(self):
        """Make sure that the queries generated in SQLAlchemy are the same as
        those precompiled into SQLite.

        """
        from allegedb.alchemy import Alchemist
        self.assertTrue(hasattr(self.engine.query, 'alchemist'))
        self.assertTrue(isinstance(self.engine.query.alchemist, Alchemist))
        from json import load
        with open(self.engine.query.json_path + '/sqlite.json', 'r') as jsonfile:
            precompiled = load(jsonfile)
        self.assertEqual(
            precompiled.keys(), self.engine.query.alchemist.sql.keys()
        )
        for (k, query) in precompiled.items():
            self.assertEqual(
                query,
                str(
                    self.engine.query.alchemist.sql[k]
                )
            )

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># defivelo-intranet -- Outil métier pour la gestion du Défi Vélo
# Copyright (C) 2016 Didier Raboud <me+defivelo@odyx.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import operator
from functools import reduce

from django.contrib.auth import get_user_model
from django.contrib.messages.views import SuccessMessageMixin
from django.db.models import Q
from django.urls import reverse_lazy
from django.utils.translation import ugettext_lazy as _
from django.views.generic import DetailView
from django.views.generic.edit import CreateView, UpdateView

from django_filters import (
    CharFilter,
    FilterSet,
    ModelMultipleChoiceFilter,
    MultipleChoiceFilter,
)
from django_filters.views import FilterView
from rolepermissions.mixins import HasPermissionsMixin

from apps.challenge.models import QualificationActivity
from apps.common import (
    DV_LANGUAGES_WITH_DEFAULT,
    DV_STATE_CHOICES_WITH_DEFAULT,
    MULTISELECTFIELD_REGEXP,
)
from apps.common.views import ExportMixin, PaginatorMixin
from defivelo.roles import DV_AVAILABLE_ROLES, has_permission, user_cantons

from .. import FORMATION_KEYS
from ..export import CollaboratorUserResource, UserResource
from ..forms import SwissDateFilter
from ..models import (
    FORMATION_CHOICES,
    USERSTATUS_ACTIVE,
    USERSTATUS_ARCHIVE,
    USERSTATUS_CHOICES,
    USERSTATUS_DELETED,
    USERSTATUS_RESERVE,
)
from .mixins import ProfileMixin, UserSelfAccessMixin

class UserDetail(UserSelfAccessMixin, ProfileMixin, DetailView):
    required_permission = "user_detail_other"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        try:
            managed_cantons = user_cantons(self.request.user)
        except LookupError:
            managed_cantons = []
        user_cantons_intersection = [
            orga.address_canton
            for orga in self.get_object().managed_organizations.all()
            if orga.address_canton in managed_cantons
        ]
        context["userprofilecanton"] = (
            user_cantons_intersection[0] if user_cantons_intersection else None
        )
        return context

class UserUpdate(UserSelfAccessMixin, ProfileMixin, SuccessMessageMixin, UpdateView):
    success_message = _("Profil mis à jour")

    def get_initial(self):
        """
        Pre-fill the form with the non-model fields
        """
        user = self.get_object()
        if hasattr(user, "profile"):
            struct = {}
            for field in self.profile_fields:
                struct[field] = getattr(user.profile, field)
            struct["actor_for"] = user.profile.actor_for.all()
            return struct

    def dispatch(self, request, *args, **kwargs):
        return super(UserUpdate, self).dispatch(request, *args, edit=True, **kwargs)

class UserCreate(HasPermissionsMixin, ProfileMixin, SuccessMessageMixin, CreateView):
    required_permission = "user_create"
    success_message = _("Utilisateur créé")

    def get_form_kwargs(self):
        kwargs = super(UserCreate, self).get_form_kwargs()
        kwargs["allow_email"] = True
        return kwargs

    def get_success_url(self):
        try:
            return reverse_lazy("user-detail", kwargs={"pk": self.object.pk})
        except Exception:
            return reverse_lazy("user-list")

class UserProfileFilterSet(FilterSet):
    def __init__(self, data=None, *args, request=None, **kwargs):
        any_filter_is_set = bool(set(self.base_filters) & set(data or {}))
        if not any_filter_is_set:
            data = {}
            for name, f in self.base_filters.items():
                initial = f.extra.get("initial")
                # filter param is either missing or empty, use initial as default
                if not data.get(name) and initial:
                    data[name] = initial
        super().__init__(data=data, *args, request=request, **kwargs)

        if not has_permission(request.user, "user_export_all_fields"):
            # That user only has a limited amount of choices available, amend them
            # Remove the inaccessible statuses
            status_choices = self.filters["profile__status"].extra["choices"]
            self.filters["profile__status"].extra["choices"] = (
                t
                for t in status_choices
                if t[0]
                not in [
                    USERSTATUS_ARCHIVE,
                    USERSTATUS_DELETED,
                ]
            )
            # Remove roles' filter, there's no reason for a collaborator to filter on roles
            del self.filters["roles"]

    def filter_multi_nonempty(queryset, name, values):
        if values:
            allor_filter = [Q(**{name: v}) for v in values if v]
            if allor_filter:
                return queryset.filter(reduce(operator.or_, allor_filter))
        return queryset

    def filter_cantons(queryset, name, values):
        if values and any(values):
            # S'il y au moins un canton en commun
            cantons_regexp = MULTISELECTFIELD_REGEXP % "|".join(
                [v for v in values if v]
            )
            allcantons_filter = [Q(profile__activity_cantons__regex=cantons_regexp)] + [
                Q(profile__affiliation_canton=v) for v in values if v
            ]
            return queryset.filter(reduce(operator.or_, allcantons_filter))
        return queryset

    def filter_languages(queryset, name, values):
        if values and any(values):
            # S'il y au moins une langue en commun
            lang_regexp = MULTISELECTFIELD_REGEXP % "|".join([v for v in values if v])
            alllangs_filter = [Q(profile__languages_challenges__regex=lang_regexp)] + [
                Q(profile__language=v) for v in values if v
            ]
            return queryset.filter(reduce(operator.or_, alllangs_filter))
        return queryset

    def filter_wide(queryset, name, value):
        if value:
            allfields_filter = [
                Q(last_name__unaccent__icontains=value),
                Q(first_name__unaccent__icontains=value),
                Q(email__icontains=value),
                Q(profile__natel__icontains=value),
            ]
            return queryset.filter(reduce(operator.or_, allfields_filter))
        return queryset

    def filter_roles(queryset, name, value):
        try:
            # Use the '0' special value (see definition of the role filter) to specially filter the absence of any role
            if int(value[0]) == 0:
                return queryset.filter(groups__isnull=True)
        except (KeyError, ValueError):
            pass
        return queryset.filter(reduce(operator.or_, [Q(groups__name=r) for r in value]))

    def filter_updated_at(queryset, name, value):
        return queryset.filter(profile__updated_at__gte=value)

    profile__language = MultipleChoiceFilter(
        label=_("Langue"),
        choices=DV_LANGUAGES_WITH_DEFAULT,
        method=filter_multi_nonempty,
    )

    profile__languages_challenges = MultipleChoiceFilter(
        label=_("Langues d’animation"),
        choices=DV_LANGUAGES_WITH_DEFAULT,
        method=filter_languages,
    )

    profile__affiliation_canton = MultipleChoiceFilter(
        label=_("Canton d’affiliation"),
        choices=DV_STATE_CHOICES_WITH_DEFAULT,
        method=filter_multi_nonempty,
    )

    profile__activity_cantons = MultipleChoiceFilter(
        label=_("Cantons"), choices=DV_STATE_CHOICES_WITH_DEFAULT, method=filter_cantons
    )
    profile__status = MultipleChoiceFilter(
        label=_("Statut"),
        choices=USERSTATUS_CHOICES,
        initial=[
            USERSTATUS_ACTIVE,
            USERSTATUS_RESERVE,
        ],
    )
    profile__formation = MultipleChoiceFilter(
        label=_("Formation"), choices=FORMATION_CHOICES, method=filter_multi_nonempty
    )
    profile__actor_for = ModelMultipleChoiceFilter(
        label=_("Intervenant"),
        queryset=(
            QualificationActivity.objects.filter(category="C").prefetch_related(
                "translations"
            )
        ),
    )

    profile__updated_at = SwissDateFilter(
        label=_("Dernière modification après le"),
        method=filter_updated_at,
    )

    roles = MultipleChoiceFilter(
        label=_("Rôle"),
        choices=tuple((t[0] if t[0] else 0, t[1]) for t in DV_AVAILABLE_ROLES),
        method=filter_roles,
    )
    q = CharFilter(label=_("Recherche"), method=filter_wide)

    class Meta:
        model = get_user_model()
        fields = [
            "profile__status",
            "profile__formation",
            "profile__actor_for",
            "profile__activity_cantons",
            "profile__updated_at",
        ]

class UserList(HasPermissionsMixin, ProfileMixin, PaginatorMixin, FilterView):
    required_permission = "user_view_list"
    filterset_class = UserProfileFilterSet
    context_object_name = "users"

class UserListExport(ExportMixin, UserList):
    export_class = UserResource()
    export_filename = _("Utilisateurs")

    def get_export_class(self, request):
        """
        From ExportMixin
        Override export_class when User has no permission
        """
        if not has_permission(self.request.user, "user_export_all_fields"):
            return CollaboratorUserResource()
        return super().get_export_class(request)

class UserDetailedList(UserList):
    paginate_by = None
    context_object_name = "users"
    template_name = "auth/user_detailed_list.html"
    page_title = _("Liste des utilisateurs")

    def get_context_data(self, **kwargs):
        context = super(UserDetailedList, self).get_context_data(**kwargs)
        # Add our page title
        context["page_title"] = self.page_title
        return context

class HelpersList(UserDetailedList):
    page_title = _("Liste des moniteurs")

    def get_queryset(self):
        return (
            super(HelpersList, self)
            .get_queryset()
            .filter(profile__formation__in=FORMATION_KEYS)
        )

class ActorsList(UserDetailedList):
    page_title = _("Liste des intervenants")

    def get_queryset(self):
        return (
            super(ActorsList, self)
            .get_queryset()
            .exclude(profile__actor_for__isnull=True)
        )
<commit_msg>DEFIVELO-238 Fix after customer review<commit_after># defivelo-intranet -- Outil métier pour la gestion du Défi Vélo
# Copyright (C) 2016 Didier Raboud <me+defivelo@odyx.org>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
import operator
from functools import reduce

from django.contrib.auth import get_user_model
from django.contrib.messages.views import SuccessMessageMixin
from django.db.models import Q
from django.urls import reverse_lazy
from django.utils.translation import ugettext_lazy as _
from django.views.generic import DetailView
from django.views.generic.edit import CreateView, UpdateView

from django_filters import (
    CharFilter,
    FilterSet,
    ModelMultipleChoiceFilter,
    MultipleChoiceFilter,
)
from django_filters.views import FilterView
from rolepermissions.mixins import HasPermissionsMixin

from apps.challenge.models import QualificationActivity
from apps.common import (
    DV_LANGUAGES_WITH_DEFAULT,
    DV_STATE_CHOICES_WITH_DEFAULT,
    MULTISELECTFIELD_REGEXP,
)
from apps.common.views import ExportMixin, PaginatorMixin
from defivelo.roles import DV_AVAILABLE_ROLES, has_permission, user_cantons

from .. import FORMATION_KEYS
from ..export import CollaboratorUserResource, UserResource
from ..forms import SwissDateFilter
from ..models import (
    FORMATION_CHOICES,
    USERSTATUS_ACTIVE,
    USERSTATUS_ARCHIVE,
    USERSTATUS_CHOICES,
    USERSTATUS_DELETED,
    USERSTATUS_RESERVE,
)
from .mixins import ProfileMixin, UserSelfAccessMixin

class UserDetail(UserSelfAccessMixin, ProfileMixin, DetailView):
    required_permission = "user_detail_other"

    def get_context_data(self, **kwargs):
        context = super().get_context_data(**kwargs)
        try:
            managed_cantons = user_cantons(self.request.user)
        except LookupError:
            managed_cantons = []
        user_cantons_intersection = [
            orga.address_canton
            for orga in self.get_object().managed_organizations.all()
            if orga.address_canton in managed_cantons
        ]
        context["userprofilecanton"] = (
            user_cantons_intersection[0] if user_cantons_intersection else None
        )
        return context

class UserUpdate(UserSelfAccessMixin, ProfileMixin, SuccessMessageMixin, UpdateView):
    success_message = _("Profil mis à jour")

    def get_initial(self):
        """
        Pre-fill the form with the non-model fields
        """
        user = self.get_object()
        if hasattr(user, "profile"):
            struct = {}
            for field in self.profile_fields:
                struct[field] = getattr(user.profile, field)
            struct["actor_for"] = user.profile.actor_for.all()
            return struct

    def dispatch(self, request, *args, **kwargs):
        return super(UserUpdate, self).dispatch(request, *args, edit=True, **kwargs)

class UserCreate(HasPermissionsMixin, ProfileMixin, SuccessMessageMixin, CreateView):
    required_permission = "user_create"
    success_message = _("Utilisateur créé")

    def get_form_kwargs(self):
        kwargs = super(UserCreate, self).get_form_kwargs()
        kwargs["allow_email"] = True
        return kwargs

    def get_success_url(self):
        try:
            return reverse_lazy("user-detail", kwargs={"pk": self.object.pk})
        except Exception:
            return reverse_lazy("user-list")

class UserProfileFilterSet(FilterSet):
    def __init__(self, data=None, *args, request=None, **kwargs):
        any_filter_is_set = bool(set(self.base_filters) & set(data or {}))
        if not any_filter_is_set:
            data = {}
            for name, f in self.base_filters.items():
                initial = f.extra.get("initial")
                # filter param is either missing or empty, use initial as default
                if not data.get(name) and initial:
                    data[name] = initial
        super().__init__(data=data, *args, request=request, **kwargs)

        if not has_permission(request.user, "user_export_all_fields"):
            # That user only has a limited amount of choices available, amend them
            # Remove the inaccessible statuses
            status_choices = self.filters["profile__status"].extra["choices"]
            self.filters["profile__status"].extra["choices"] = (
                t
                for t in status_choices
                if t[0]
                not in [
                    USERSTATUS_ARCHIVE,
                    USERSTATUS_DELETED,
                ]
            )
            # Remove filters that should be displayed to admins only
            del self.filters["roles"]
            del self.filters["profile__updated_at"]

    def filter_multi_nonempty(queryset, name, values):
        if values:
            allor_filter = [Q(**{name: v}) for v in values if v]
            if allor_filter:
                return queryset.filter(reduce(operator.or_, allor_filter))
        return queryset

    def filter_cantons(queryset, name, values):
        if values and any(values):
            # S'il y au moins un canton en commun
            cantons_regexp = MULTISELECTFIELD_REGEXP % "|".join(
                [v for v in values if v]
            )
            allcantons_filter = [Q(profile__activity_cantons__regex=cantons_regexp)] + [
                Q(profile__affiliation_canton=v) for v in values if v
            ]
            return queryset.filter(reduce(operator.or_, allcantons_filter))
        return queryset

    def filter_languages(queryset, name, values):
        if values and any(values):
            # S'il y au moins une langue en commun
            lang_regexp = MULTISELECTFIELD_REGEXP % "|".join([v for v in values if v])
            alllangs_filter = [Q(profile__languages_challenges__regex=lang_regexp)] + [
                Q(profile__language=v) for v in values if v
            ]
            return queryset.filter(reduce(operator.or_, alllangs_filter))
        return queryset

    def filter_wide(queryset, name, value):
        if value:
            allfields_filter = [
                Q(last_name__unaccent__icontains=value),
                Q(first_name__unaccent__icontains=value),
                Q(email__icontains=value),
                Q(profile__natel__icontains=value),
            ]
            return queryset.filter(reduce(operator.or_, allfields_filter))
        return queryset

    def filter_roles(queryset, name, value):
        try:
            # Use the '0' special value (see definition of the role filter) to specially filter the absence of any role
            if int(value[0]) == 0:
                return queryset.filter(groups__isnull=True)
        except (KeyError, ValueError):
            pass
        return queryset.filter(reduce(operator.or_, [Q(groups__name=r) for r in value]))

    def filter_updated_at(queryset, name, value):
        return queryset.filter(profile__updated_at__gte=value)

    profile__language = MultipleChoiceFilter(
        label=_("Langue"),
        choices=DV_LANGUAGES_WITH_DEFAULT,
        method=filter_multi_nonempty,
    )

    profile__languages_challenges = MultipleChoiceFilter(
        label=_("Langues d’animation"),
        choices=DV_LANGUAGES_WITH_DEFAULT,
        method=filter_languages,
    )

    profile__affiliation_canton = MultipleChoiceFilter(
        label=_("Canton d’affiliation"),
        choices=DV_STATE_CHOICES_WITH_DEFAULT,
        method=filter_multi_nonempty,
    )

    profile__activity_cantons = MultipleChoiceFilter(
        label=_("Cantons"), choices=DV_STATE_CHOICES_WITH_DEFAULT, method=filter_cantons
    )
    profile__status = MultipleChoiceFilter(
        label=_("Statut"),
        choices=USERSTATUS_CHOICES,
        initial=[
            USERSTATUS_ACTIVE,
            USERSTATUS_RESERVE,
        ],
    )
    profile__formation = MultipleChoiceFilter(
        label=_("Formation"), choices=FORMATION_CHOICES, method=filter_multi_nonempty
    )
    profile__actor_for = ModelMultipleChoiceFilter(
        label=_("Intervenant"),
        queryset=(
            QualificationActivity.objects.filter(category="C").prefetch_related(
                "translations"
            )
        ),
    )

    profile__updated_at = SwissDateFilter(
        label=_("Dernière modification après le"),
        method=filter_updated_at,
    )

    roles = MultipleChoiceFilter(
        label=_("Rôle"),
        choices=tuple((t[0] if t[0] else 0, t[1]) for t in DV_AVAILABLE_ROLES),
        method=filter_roles,
    )
    q = CharFilter(label=_("Recherche"), method=filter_wide)

    class Meta:
        model = get_user_model()
        fields = [
            "profile__status",
            "profile__formation",
            "profile__actor_for",
            "profile__activity_cantons",
            "profile__updated_at",
        ]

class UserList(HasPermissionsMixin, ProfileMixin, PaginatorMixin, FilterView):
    required_permission = "user_view_list"
    filterset_class = UserProfileFilterSet
    context_object_name = "users"

class UserListExport(ExportMixin, UserList):
    export_class = UserResource()
    export_filename = _("Utilisateurs")

    def get_export_class(self, request):
        """
        From ExportMixin
        Override export_class when User has no permission
        """
        if not has_permission(self.request.user, "user_export_all_fields"):
            return CollaboratorUserResource()
        return super().get_export_class(request)

class UserDetailedList(UserList):
    paginate_by = None
    context_object_name = "users"
    template_name = "auth/user_detailed_list.html"
    page_title = _("Liste des utilisateurs")

    def get_context_data(self, **kwargs):
        context = super(UserDetailedList, self).get_context_data(**kwargs)
        # Add our page title
        context["page_title"] = self.page_title
        return context

class HelpersList(UserDetailedList):
    page_title = _("Liste des moniteurs")

    def get_queryset(self):
        return (
            super(HelpersList, self)
            .get_queryset()
            .filter(profile__formation__in=FORMATION_KEYS)
        )

class ActorsList(UserDetailedList):
    page_title = _("Liste des intervenants")

    def get_queryset(self):
        return (
            super(ActorsList, self)
            .get_queryset()
            .exclude(profile__actor_for__isnull=True)
        )
<|endoftext|>


<commit_before>__version__ = '0.1.11'
__author__ = 'eric@civicknowledge.com'
<commit_msg>Version bump. #25.<commit_after>__version__ = '0.1.12'
__author__ = 'eric@civicknowledge.com'
<|endoftext|>


<commit_before>#
# Copyright (c) 2007 rPath, Inc.
#

import socket
from urllib import urlopen

from amiconfig.errors import *

class InstanceData:
    def __init__(self):
        self.urlbase = 'http://169.254.169.254/1.0/'

    def open(self, path):
        try:
            results = urlopen('%s/%s' % (self.urlbase, path))
        except Exception, e:
            raise EC2DataRetrievalError, '[Errno %s] %s' % (e.errno, e.strerror)
        if results.headers.gettype() == 'text/html':
            raise EC2DataRetrievalError, '%s' % results.read()
        return results

    def read(self, path):
        return self.open(path).read()

    def getUserData(self):
        return self.read('user-data')

    def getAMIId(self):
        return self.read('meta-data/ami-id')

    def getAMILaunchIndex(self):
        return self.read('meta-data/ami-launch-index')

    def getAMIManifestPath(self):
        return self.read('meta-data/ami-manifest-path')

    def getHostname(self):
        return self.read('meta-data/hostname')

    def getInstanceId(self):
        return self.read('meta-data/instance-id')

    def getLocalIPv4(self):
        return self.read('meta-data/local-ipv4')

    def getReservationId(self):
        return self.read('meta-data/reservation-id')

    def getSecurityGroups(self):
        list = []
        for group in self.open('meta-data/security-groups'):
            list.append(group.strip())
        return list

    def getKeyIdList(self):
        list = []
        for key in self.open('meta-data/public-keys/'):
            lst = key.split('=')
            if len(lst) in [0, 1]:
                continue
            id = lst[0]
            name = '='.join(lst[1:])
            list.append((id, name))
        return list

    def getSSHKey(self, id=0):
        return self.read('meta-data/public-keys/0/openssh-key')
<commit_msg>let consumers secify version<commit_after>#
# Copyright (c) 2007 rPath, Inc.
#

import socket
from urllib import urlopen

from amiconfig.errors import *

class InstanceData:
    apiversion = '1.0'

    def __init__(self):
        self.urlbase = 'http://169.254.169.254/%s/' % self.apiversion

    def open(self, path):
        try:
            results = urlopen('%s/%s' % (self.urlbase, path))
        except Exception, e:
            raise EC2DataRetrievalError, '[Errno %s] %s' % (e.errno, e.strerror)
        if results.headers.gettype() == 'text/html':
            raise EC2DataRetrievalError, '%s' % results.read()
        return results

    def read(self, path):
        return self.open(path).read()

    def getUserData(self):
        return self.read('user-data')

    def getAMIId(self):
        return self.read('meta-data/ami-id')

    def getAMILaunchIndex(self):
        return self.read('meta-data/ami-launch-index')

    def getAMIManifestPath(self):
        return self.read('meta-data/ami-manifest-path')

    def getHostname(self):
        return self.read('meta-data/hostname')

    def getInstanceId(self):
        return self.read('meta-data/instance-id')

    def getLocalIPv4(self):
        return self.read('meta-data/local-ipv4')

    def getReservationId(self):
        return self.read('meta-data/reservation-id')

    def getSecurityGroups(self):
        list = []
        for group in self.open('meta-data/security-groups'):
            list.append(group.strip())
        return list

    def getKeyIdList(self):
        list = []
        for key in self.open('meta-data/public-keys/'):
            lst = key.split('=')
            if len(lst) in [0, 1]:
                continue
            id = lst[0]
            name = '='.join(lst[1:])
            list.append((id, name))
        return list

    def getSSHKey(self, id=0):
        return self.read('meta-data/public-keys/0/openssh-key')
<|endoftext|>