{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.13.15","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":5,"nbformat":4,"cells":[{"id":"cell-00","cell_type":"markdown","source":"# LLMSQL subset selection\n\nProfiles, validates and selects LLMSQL train questions whose gold SQL references column names that need quotes (spaces, special characters, leading digits, reserved words), to join BIRD + Spider in the v2 fine-tune of Qwen3.5-4B and Qwen3.5-9B. Quoting is compared against BIRD and Spider train, and the selected SQL is rewritten to quote identifiers with `IDENTIFIER_QUOTE`.\n\n- **Runs on:** Kaggle, CPU session with Internet enabled and an `HF_TOKEN` secret (no GPU needed)\n- **Python:** 3.11\n- **Last adjusted:** 2026","metadata":{}},{"id":"cell-01","cell_type":"code","source":"%pip install -q -U \"transformers>=5.5\" datasets","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:37:36.762668Z","iopub.execute_input":"2026-10-20T00:37:36.762973Z","iopub.status.idle":"2026-10-20T00:37:58.613807Z","shell.execute_reply.started":"2026-10-20T00:37:36.762950Z","shell.execute_reply":"2026-10-20T00:37:58.613073Z"}},"outputs":[{"name":"stdout","text":"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.8/12.8 MB\u001b[0m \u001b[31m86.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m604.4/604.4 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.9/53.9 MB\u001b[0m \u001b[31m35.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m:00:01\u001b[0m00:01\u001b[0m\n\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m846.4/846.4 kB\u001b[0m \u001b[31m28.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25h\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\nbigframes 2.48.0 requires google-cloud-bigquery-storage<3.0.0,>=2.30.0, which is not installed.\u001b[0m\u001b[31m\n\u001b[0mNote: you may need to restart the kernel to use updated packages.\n","output_type":"stream"}],"execution_count":1},{"id":"cell-02","cell_type":"code","source":"import hashlib\nimport os\nimport re\nimport sqlite3\nimport subprocess\nimport sys\nimport time\nimport zipfile\nfrom collections import Counter\nfrom functools import lru_cache\nfrom pathlib import Path\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport pandas as pd\nfrom datasets import load_dataset\nfrom huggingface_hub import hf_hub_download\nfrom IPython.display import display\nfrom kaggle_secrets import UserSecretsClient\nfrom tqdm.auto import tqdm\nfrom transformers import AutoTokenizer\n\nPath(\"requirements.txt\").write_text(\n subprocess.run([sys.executable, \"-m\", \"pip\", \"freeze\"], capture_output=True, text=True).stdout\n)\npd.set_option(\"display.max_colwidth\", 120)\nos.environ[\"HF_TOKEN\"] = UserSecretsClient().get_secret(\"HF_TOKEN\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:37:58.615776Z","iopub.execute_input":"2026-10-20T00:37:58.615988Z","iopub.status.idle":"2026-10-20T00:38:20.823866Z","shell.execute_reply.started":"2026-10-20T00:37:58.615970Z","shell.execute_reply":"2026-10-20T00:38:20.822755Z"}},"outputs":[],"execution_count":2},{"id":"cell-03","cell_type":"code","source":"LLMSQL_REPO = \"llmsql-bench/llmsql-benchmark\"\nBIRD_TRAIN_REPO = \"EuricoGVP/birdsql_complete_trainset\"\nSPIDER_REPO = \"EuricoGVP/spider_sql_complete\"\nSPIDER_EXCLUDED_DBS = [\"formula_1\"]\nTOKENIZER_NAMES = [\"Qwen/Qwen3.5-4B\", \"Qwen/Qwen3.5-9B\"]\n\nDATA_DIR = Path(\"/tmp/llmsql\")\nOUTPUT_DIR = Path(\"/kaggle/working\")\n\nMAX_SEQ_LENGTH = 4096\nTARGET_SIZE = 1000\nREASON_QUOTAS = {\"reserved_word\": 250, \"leading_digit\": 250}\nIDENTIFIER_QUOTE = \"`\"\nSQL_TIMEOUT_SECONDS = 5\nSEED = 42\n\nQUOTING_REASONS = [\"space\", \"special_character\", \"leading_digit\", \"reserved_word\"]\nQUOTE_PATTERNS = {\"backtick\": \"`{}`\", \"double_quote\": '\"{}\"', \"bracket\": r\"\\[{}\\]\"}\nDATASET_COLORS = {\"BIRD\": \"#2a78d6\", \"Spider\": \"#eb6834\", \"LLMSQL\": \"#1baf7a\"}\n\nSYSTEM_PROMPT = (\n \"You are an expert SQLite data analyst.\\n\"\n \"Rules:\\n\"\n \"- Use only the tables and columns that exist in the schema.\\n\"\n \"- If a hint is provided, use it.\\n\"\n \"- Return exactly the columns the question asks for.\\n\"\n \"- Output only the SQL query inside a single ```sql code block, with no explanation.\"\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:38:20.825018Z","iopub.execute_input":"2026-10-20T00:38:20.825572Z","iopub.status.idle":"2026-10-20T00:38:20.830008Z","shell.execute_reply.started":"2026-10-20T00:38:20.825552Z","shell.execute_reply":"2026-10-20T00:38:20.829430Z"}},"outputs":[],"execution_count":3},{"id":"cell-04","cell_type":"markdown","source":"## 1. Download\n\n`llmsql-benchmark` is the cleaned WikiSQL: one table per question, every identifier already in double quotes, and a single `sqlite_tables.db` with all tables. Only the train split is used, so validation and test stay untouched.","metadata":{}},{"id":"cell-05","cell_type":"code","source":"DATA_DIR.mkdir(parents=True, exist_ok=True)\nquestions_path = Path(hf_hub_download(LLMSQL_REPO, \"train_questions.jsonl\", repo_type=\"dataset\", local_dir=DATA_DIR))\ndb_path = Path(hf_hub_download(LLMSQL_REPO, \"sqlite_tables.db\", repo_type=\"dataset\", local_dir=DATA_DIR))\n\nquestions = pd.read_json(questions_path, lines=True, dtype={\"table_id\": str})\nllmsql_connection = sqlite3.connect(f\"file:{db_path}?mode=ro\", uri=True)\n\ndisplay(pd.DataFrame({\n \"questions\": [len(questions)],\n \"tables\": [questions[\"table_id\"].nunique()],\n \"avg_questions_per_table\": [len(questions) / questions[\"table_id\"].nunique()],\n}).round(2))\ndisplay(questions.head(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:38:20.830833Z","iopub.execute_input":"2026-10-20T00:38:20.831131Z","iopub.status.idle":"2026-10-20T00:38:32.170499Z","shell.execute_reply.started":"2026-10-20T00:38:20.831096Z","shell.execute_reply":"2026-10-20T00:38:32.169765Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"train_questions.jsonl: reconstructing file: 0%| | 0.00B / 12.8MB ","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"165e27d5114442178181834c04231c7b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"train_questions.jsonl: downloading bytes: | 0.00B ","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e020b3c790ef45dbab0db118d4447e1b"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"sqlite_tables.db: reconstructing file: 0%| | 0.00B / 118MB ","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"84b5b2e7050349109ce539751d369365"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"sqlite_tables.db: downloading bytes: | 0.00B ","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f788ddf9a210406ca55705d5e320b6f3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":" questions tables avg_questions_per_table\n0 56111 17935 3.13","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
questionstablesavg_questions_per_table
056111179353.13
\n
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":" question_id table_id \\\n0 24220 1-1000181-1 \n1 24221 1-1000181-1 \n2 24222 1-1000181-1 \n\n question \\\n0 Tell me what the notes are for SOUTH AUSTRALIA \n1 What is the current series where the New series began in June 2011? \n2 What is the format for South Australia? \n\n sql \n0 SELECT \"Notes\" FROM \"1-1000181-1\" WHERE \"Current slogan\" = 'SOUTH AUSTRALIA'; \n1 SELECT \"Current series\" FROM \"1-1000181-1\" WHERE \"Notes\" = 'New series began in June 2011'; \n2 SELECT \"Format\" FROM \"1-1000181-1\" WHERE \"State/territory\" = 'South Australia'; ","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
question_idtable_idquestionsql
0242201-1000181-1Tell me what the notes are for SOUTH AUSTRALIASELECT \"Notes\" FROM \"1-1000181-1\" WHERE \"Current slogan\" = 'SOUTH AUSTRALIA';
1242211-1000181-1What is the current series where the New series began in June 2011?SELECT \"Current series\" FROM \"1-1000181-1\" WHERE \"Notes\" = 'New series began in June 2011';
2242221-1000181-1What is the format for South Australia?SELECT \"Format\" FROM \"1-1000181-1\" WHERE \"State/territory\" = 'South Australia';
\n
"},"metadata":{}}],"execution_count":4},{"id":"cell-06","cell_type":"markdown","source":"## 2. Table profile\n\nEach table gets its DDL, size and the columns that need quotes. As in the SynSQL notebook, a name needs quotes when SQLite fails to read it unquoted, and each such name gets a reason. Table names are left out of the criterion, since every LLMSQL table id (like `1-1000181-1`) needs quotes.","metadata":{}},{"id":"cell-07","cell_type":"code","source":"quote_check_connection = sqlite3.connect(\":memory:\")\n\n\ndef quote_name(name):\n return '\"' + name.replace('\"', '\"\"') + '\"'\n\n\n@lru_cache(maxsize=None)\ndef needs_quoting(identifier):\n try:\n row = quote_check_connection.execute(\n f\"SELECT {identifier} FROM (SELECT 'sentinel' AS {quote_name(identifier)})\"\n ).fetchone()\n except (sqlite3.Error, sqlite3.Warning):\n return True\n return row[0] != \"sentinel\"\n\n\ndef quoting_reason(identifier):\n if not needs_quoting(identifier):\n return None\n if \" \" in identifier:\n return \"space\"\n if re.search(r\"[^A-Za-z0-9_]\", identifier):\n return \"special_character\"\n if identifier[:1].isdigit():\n return \"leading_digit\"\n return \"reserved_word\"\n\n\ndef profile_table(table_id):\n row = llmsql_connection.execute(\n \"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?\", (table_id,)\n ).fetchone()\n if row is None:\n return None\n columns = [column[1] for column in llmsql_connection.execute(f\"PRAGMA table_info({quote_name(table_id)})\")]\n return {\n \"table_id\": table_id,\n \"ddl\": row[0],\n \"n_columns\": len(columns),\n \"n_rows\": llmsql_connection.execute(f\"SELECT COUNT(*) FROM {quote_name(table_id)}\").fetchone()[0],\n \"quoted_columns\": sorted({column for column in columns if needs_quoting(column)}),\n }\n\n\nprofiles = [profile_table(table_id) for table_id in tqdm(questions[\"table_id\"].unique(), desc=\"Profiling tables\")]\ntables = pd.DataFrame([profile for profile in profiles if profile is not None]).set_index(\"table_id\")\nprint(f\"Tables missing from sqlite_tables.db: {sum(profile is None for profile in profiles)}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:38:32.172803Z","iopub.execute_input":"2026-10-20T00:38:32.173055Z","iopub.status.idle":"2026-10-20T00:39:43.123809Z","shell.execute_reply.started":"2026-10-20T00:38:32.173014Z","shell.execute_reply":"2026-10-20T00:39:43.122866Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"Profiling tables: 0%| | 0/17935 [00:00 0).mean():.1%}\")\n\nquoted_column_names = [column for columns in tables[\"quoted_columns\"] for column in columns]\ndisplay(pd.Series([quoting_reason(column) for column in quoted_column_names]).value_counts().rename(\"columns\").to_frame())\ndisplay(pd.DataFrame(Counter(quoted_column_names).most_common(15), columns=[\"column\", \"tables\"]))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:39:43.124965Z","iopub.execute_input":"2026-10-20T00:39:43.125330Z","iopub.status.idle":"2026-10-20T00:39:43.188462Z","shell.execute_reply.started":"2026-10-20T00:39:43.125300Z","shell.execute_reply":"2026-10-20T00:39:43.187444Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":" n_columns n_rows\ncount 17935.0 17935.0\nmean 6.3 17.1\nstd 1.7 29.6\nmin 5.0 5.0\n25% 5.0 7.0\n50% 6.0 11.0\n75% 7.0 18.0\nmax 44.0 1956.0","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
n_columnsn_rows
count17935.017935.0
mean6.317.1
std1.729.6
min5.05.0
25%5.07.0
50%6.011.0
75%7.018.0
max44.01956.0
\n
"},"metadata":{}},{"name":"stdout","text":"Tables with at least one column that needs quotes: 75.5%\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":" columns\nspace 37156\nspecial_character 3913\nleading_digit 1738\nreserved_word 157","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
columns
space37156
special_character3913
leading_digit1738
reserved_word157
\n
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":" column tables\n0 Home team 942\n1 Away team 929\n2 Away team score 746\n3 Home team score 746\n4 High points 487\n5 To par 485\n6 High rebounds 476\n7 High assists 469\n8 Location Attendance 466\n9 Directed by 455\n10 Written by 444\n11 Original air date 439\n12 Time/Retired 311\n13 First elected 298\n14 No. in series 268","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
columntables
0Home team942
1Away team929
2Away team score746
3Home team score746
4High points487
5To par485
6High rebounds476
7High assists469
8Location Attendance466
9Directed by455
10Written by444
11Original air date439
12Time/Retired311
13First elected298
14No. in series268
\n
"},"metadata":{}}],"execution_count":6},{"id":"cell-09","cell_type":"markdown","source":"## 3. Tokenizers\n\nSame check as the SynSQL notebook: the tokenizer files of the 4B and 9B are hashed, identical files are counted once, and different ones make every count take the maximum. Examples are rendered like `birdsql-fine-tunning.ipynb`, and `SYSTEM_PROMPT` is a reconstruction of the training prompt.","metadata":{}},{"id":"cell-10","cell_type":"code","source":"def tokenizer_fingerprint(model_name):\n files = [\"tokenizer.json\", \"tokenizer_config.json\", \"chat_template.jinja\"]\n return tuple(hashlib.sha256(Path(hf_hub_download(model_name, file)).read_bytes()).hexdigest() for file in files)\n\n\nfingerprints = {name: tokenizer_fingerprint(name) for name in TOKENIZER_NAMES}\ntokenizer_groups = {}\nfor name, fingerprint in fingerprints.items():\n tokenizer_groups.setdefault(fingerprint, []).append(name)\ntokenizers = {\" + \".join(names): AutoTokenizer.from_pretrained(names[0]) for names in tokenizer_groups.values()}\n\ndisplay(pd.DataFrame({\n \"model\": list(fingerprints),\n \"tokenizer_group\": [\" + \".join(tokenizer_groups[fingerprint]) for fingerprint in fingerprints.values()],\n}))\nprint(f\"{len(tokenizers)} distinct tokenizer(s) to count with\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:39:43.189356Z","iopub.execute_input":"2026-10-20T00:39:43.189610Z","iopub.status.idle":"2026-10-20T00:39:47.029354Z","shell.execute_reply.started":"2026-10-20T00:39:43.189590Z","shell.execute_reply":"2026-10-20T00:39:47.028361Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"tokenizer.json: reconstructing file: 0%| | 0.00B / 12.8MB ","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"388894e663b5496ab47c86378637873a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json: downloading bytes: | 0.00B ","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e270ecf9638744b1a0eb97aa835419a1"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json: 0%| | 0.00/16.7k [00:00\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
modeltokenizer_group
0Qwen/Qwen3.5-4BQwen/Qwen3.5-4B + Qwen/Qwen3.5-9B
1Qwen/Qwen3.5-9BQwen/Qwen3.5-4B + Qwen/Qwen3.5-9B
\n"},"metadata":{}},{"name":"stdout","text":"1 distinct tokenizer(s) to count with\n","output_type":"stream"}],"execution_count":7},{"id":"cell-11","cell_type":"code","source":"def render_example(tokenizer, ddl, hint, question, sql):\n hint = hint if hint and hint.strip() else \"None\"\n messages = [\n {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n {\"role\": \"user\", \"content\": f\"Database schema:\\n{ddl}\\n\\nHint: {hint}\\n\\nQuestion: {question}\"},\n ]\n prompt = tokenizer.apply_chat_template(\n messages, tokenize=False, add_generation_prompt=True, enable_thinking=False\n )\n return f\"{prompt}```sql\\n{sql}\\n```<|im_end|>\"\n\n\ndef count_tokens(tokenizer, texts, batch_size=2048):\n lengths = []\n for start in tqdm(range(0, len(texts), batch_size), desc=\"Tokenizing\", leave=False):\n batch = tokenizer(texts[start:start + batch_size], add_special_tokens=False)[\"input_ids\"]\n lengths.extend(len(ids) for ids in batch)\n return np.array(lengths)\n\n\ndef count_example_tokens(ddls, hints, questions, sqls):\n rows = list(zip(ddls, hints, questions, sqls))\n return np.max(\n [count_tokens(tokenizer, [render_example(tokenizer, *row) for row in rows]) for tokenizer in tokenizers.values()],\n axis=0,\n )\n\n\ndef token_summary(name, lengths):\n return {\n \"dataset\": name,\n \"rows\": len(lengths),\n \"p50\": np.median(lengths),\n \"p75\": np.quantile(lengths, 0.75),\n \"p95\": np.quantile(lengths, 0.95),\n \"max\": lengths.max(),\n \"over_max_seq_length\": (lengths > MAX_SEQ_LENGTH).mean(),\n }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:39:47.030326Z","iopub.execute_input":"2026-10-20T00:39:47.030634Z","iopub.status.idle":"2026-10-20T00:39:47.040849Z","shell.execute_reply.started":"2026-10-20T00:39:47.030604Z","shell.execute_reply":"2026-10-20T00:39:47.039756Z"}},"outputs":[],"execution_count":8},{"id":"cell-12","cell_type":"markdown","source":"## 4. Reference: quoting in BIRD and Spider\n\nThe BIRD and Spider DDLs are loaded into an in-memory SQLite to list their tables and columns, and each gold query is checked for quoted references to names that need quotes. This shows how much quoting signal the base mix already has, and which quote style it uses. For LLMSQL only columns count, since every LLMSQL table id needs quotes and would put every row at 100%.","metadata":{}},{"id":"cell-13","cell_type":"code","source":"def schema_quoted_identifiers(ddl):\n connection = sqlite3.connect(\":memory:\")\n statements = [statement.strip().rstrip(\";\") for statement in re.split(r\"(?i)(?=\\bCREATE\\s+TABLE\\b)\", ddl) if statement.strip()]\n n_failed = 0\n for statement in statements:\n try:\n connection.execute(statement)\n except (sqlite3.Error, sqlite3.Warning):\n n_failed += 1\n table_names = [row[0] for row in connection.execute(\"SELECT name FROM sqlite_master WHERE type = 'table'\")]\n identifiers = set(table_names) | {\n row[1] for table in table_names for row in connection.execute(f\"PRAGMA table_info({quote_name(table)})\")\n }\n connection.close()\n return {\"quoted_identifiers\": sorted(name for name in identifiers if needs_quoting(name)), \"n_failed\": n_failed}\n\n\ndef quoted_references(sql, identifiers):\n names = set()\n styles = set()\n for name in identifiers:\n for style, pattern in QUOTE_PATTERNS.items():\n if re.search(\"(?i)\" + pattern.format(re.escape(name)), sql):\n names.add(name)\n styles.add(style)\n return sorted(names), sorted(styles)\n\n\ndef quoting_summary(name, quoted_identifiers, quote_styles):\n has_quoted = quoted_identifiers.str.len() > 0\n reasons = quoted_identifiers.apply(lambda names: {quoting_reason(identifier) for identifier in names})\n return {\n \"dataset\": name,\n \"rows\": len(quoted_identifiers),\n \"with_quoted_identifier\": has_quoted.mean(),\n **{f\"with_{reason}\": reasons.apply(lambda found: reason in found).mean() for reason in QUOTING_REASONS},\n **{f\"uses_{style}\": quote_styles[has_quoted].apply(lambda found: style in found).mean() for style in QUOTE_PATTERNS},\n }","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:39:47.041837Z","iopub.execute_input":"2026-10-20T00:39:47.042482Z","iopub.status.idle":"2026-10-20T00:39:47.074617Z","shell.execute_reply.started":"2026-10-20T00:39:47.042450Z","shell.execute_reply":"2026-10-20T00:39:47.073806Z"}},"outputs":[],"execution_count":9},{"id":"cell-14","cell_type":"code","source":"bird = load_dataset(BIRD_TRAIN_REPO, \"questions\", split=\"train\").to_pandas()\nbird_schemas = {row[\"db_id\"]: row[\"schema\"] for row in load_dataset(BIRD_TRAIN_REPO, \"schemas\", split=\"train\")}\nbird[\"schema\"] = bird[\"db_id\"].map(bird_schemas)\nbird[\"evidence\"] = bird[\"evidence\"].fillna(\"\")\n\nspider = load_dataset(SPIDER_REPO, \"questions\", split=\"train\").to_pandas()\nspider = spider[~spider[\"db_id\"].isin(SPIDER_EXCLUDED_DBS)].copy()\nspider_schemas = {row[\"db_id\"]: row[\"schema_raw\"] for row in load_dataset(SPIDER_REPO, \"schemas\", split=\"train\")}\nspider[\"schema\"] = spider[\"db_id\"].map(spider_schemas)\nspider[\"evidence\"] = \"\"\n\nfor name, frame, schemas in [(\"BIRD\", bird, bird_schemas), (\"Spider\", spider, spider_schemas)]:\n parsed = {db_id: schema_quoted_identifiers(ddl) for db_id, ddl in schemas.items()}\n print(f\"{name}: DDL statements that failed to load: {sum(info['n_failed'] for info in parsed.values())}\")\n references = [quoted_references(sql, parsed[db_id][\"quoted_identifiers\"]) for db_id, sql in zip(frame[\"db_id\"], frame[\"sql\"])]\n frame[\"quoted_identifiers\"] = [names for names, _ in references]\n frame[\"quote_styles\"] = [styles for _, styles in references]\n frame[\"total_tokens\"] = count_example_tokens(frame[\"schema\"], frame[\"evidence\"], frame[\"question\"], frame[\"sql\"])\n\ndisplay(pd.DataFrame([\n quoting_summary(\"BIRD train\", bird[\"quoted_identifiers\"], bird[\"quote_styles\"]),\n quoting_summary(\"Spider train\", spider[\"quoted_identifiers\"], spider[\"quote_styles\"]),\n]).round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:39:47.075637Z","iopub.execute_input":"2026-10-20T00:39:47.075865Z","iopub.status.idle":"2026-10-20T00:40:10.975743Z","shell.execute_reply.started":"2026-10-20T00:39:47.075846Z","shell.execute_reply":"2026-10-20T00:40:10.975106Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"README.md: 0%| | 0.00/4.07k [00:00\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
datasetrowswith_quoted_identifierwith_spacewith_special_characterwith_leading_digitwith_reserved_worduses_backtickuses_double_quoteuses_bracket
0BIRD train65970.0740.0470.0190.00.0080.9920.010.0
1Spider train85760.0000.0000.0000.00.000NaNNaNNaN
\n"},"metadata":{}}],"execution_count":10},{"id":"cell-15","cell_type":"markdown","source":"## 5. LLMSQL candidates\n\nA question is a candidate when its gold SQL references at least one column that needs quotes. The aggregation (`COUNT`, `MAX`, ...) is extracted to check later that the selection keeps a varied mix.","metadata":{}},{"id":"cell-16","cell_type":"code","source":"questions = questions[questions[\"table_id\"].isin(tables.index)].copy()\nreferences = [\n quoted_references(sql, tables.at[table_id, \"quoted_columns\"])\n for table_id, sql in tqdm(zip(questions[\"table_id\"], questions[\"sql\"]), total=len(questions), desc=\"Matching columns\")\n]\nquestions[\"quoted_identifiers\"] = [names for names, _ in references]\nquestions[\"quote_styles\"] = [styles for _, styles in references]\nquestions[\"quoting_reasons\"] = questions[\"quoted_identifiers\"].apply(\n lambda names: sorted({quoting_reason(identifier) for identifier in names})\n)\nquestions[\"has_space_column\"] = questions[\"quoting_reasons\"].apply(lambda reasons: \"space\" in reasons)\nquestions[\"aggregation\"] = (\n questions[\"sql\"].str.extract(r\"(?i)^\\s*SELECT\\s+(MAX|MIN|COUNT|SUM|AVG)\\s*\\(\", expand=False).str.upper().fillna(\"NONE\")\n)\n\ncandidates = questions[questions[\"quoted_identifiers\"].str.len() > 0].copy()\nprint(f\"{len(candidates):,} candidate questions ({len(candidates) / len(questions):.1%}) \"\n f\"from {candidates['table_id'].nunique():,} tables\")\ndisplay(pd.DataFrame([\n quoting_summary(\"LLMSQL train\", questions[\"quoted_identifiers\"], questions[\"quote_styles\"]),\n]).round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:40:10.976547Z","iopub.execute_input":"2026-10-20T00:40:10.977300Z","iopub.status.idle":"2026-10-20T00:40:15.735403Z","shell.execute_reply.started":"2026-10-20T00:40:10.977276Z","shell.execute_reply":"2026-10-20T00:40:15.734172Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"Matching columns: 0%| | 0/56111 [00:00\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
datasetrowswith_quoted_identifierwith_spacewith_special_characterwith_leading_digitwith_reserved_worduses_backtickuses_double_quoteuses_bracket
0LLMSQL train561110.5730.5140.0640.0150.0030.01.00.0
\n"},"metadata":{}}],"execution_count":11},{"id":"cell-17","cell_type":"markdown","source":"## 6. Validation and requoting\n\nEach candidate runs twice on `sqlite_tables.db`: as published, and with every double-quoted identifier rewritten to `IDENTIFIER_QUOTE`. The SQL is scanned left to right, so string literals in single quotes are left alone and identifiers with apostrophes (like `\"Women's doubles\"`) are rewritten whole. A candidate is kept only if both versions run and return the same rows, and the result is not empty. NULL and zero results also count as empty, to stay on the safe side with the case-mismatch problems LLMSQL inherited from WikiSQL.","metadata":{}},{"id":"cell-18","cell_type":"code","source":"quoted_token_pattern = re.compile(\"'(?:[^']|'')*'\" + '|\"(?:[^\"]|\"\")*\"')\n\n\ndef requote_identifiers(sql, quote):\n def replace(match):\n token = match.group(0)\n if token.startswith(\"'\"):\n return token\n return quote + token[1:-1].replace('\"\"', '\"').replace(quote, quote * 2) + quote\n\n return quoted_token_pattern.sub(replace, sql)\n\n\ndef has_double_quoted_identifier(sql):\n return any(token.startswith('\"') for token in quoted_token_pattern.findall(sql))\n\n\ndef execute_sql(sql):\n deadline = time.monotonic() + SQL_TIMEOUT_SECONDS\n llmsql_connection.set_progress_handler(lambda: int(time.monotonic() > deadline), 10_000)\n try:\n return llmsql_connection.execute(sql).fetchall()\n except (sqlite3.Error, sqlite3.Warning):\n return None\n\n\ncandidates[\"sql_requoted\"] = [requote_identifiers(sql, IDENTIFIER_QUOTE) for sql in candidates[\"sql\"]]\noriginal_results = [execute_sql(sql) for sql in tqdm(candidates[\"sql\"], desc=\"Executing original SQL\")]\nrequoted_results = [execute_sql(sql) for sql in tqdm(candidates[\"sql_requoted\"], desc=\"Executing requoted SQL\")]\n\ncandidates[\"executes\"] = [rows is not None for rows in original_results]\ncandidates[\"requote_matches\"] = [\n original is not None and requoted is not None and sorted(map(repr, original)) == sorted(map(repr, requoted))\n for original, requoted in zip(original_results, requoted_results)\n]\ncandidates[\"n_result_rows\"] = [len(rows) if rows is not None else 0 for rows in original_results]\ncandidates[\"is_non_empty\"] = [\n rows is not None and any(value not in (None, 0) for row in rows for value in row) for rows in original_results\n]\n\ndisplay(pd.DataFrame({\n \"candidates\": [len(candidates)],\n \"executes\": [candidates[\"executes\"].mean()],\n \"requote_matches\": [candidates[\"requote_matches\"].mean()],\n \"is_non_empty\": [candidates[\"is_non_empty\"].mean()],\n}).round(3))\nprint(f\"Requoted SQL that still has double-quoted identifiers: {candidates['sql_requoted'].apply(has_double_quoted_identifier).sum()}\")\ndisplay(candidates.loc[candidates[\"executes\"] & ~candidates[\"requote_matches\"], [\"sql\", \"sql_requoted\"]].head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:40:15.736825Z","iopub.execute_input":"2026-10-20T00:40:15.737142Z","iopub.status.idle":"2026-10-20T00:40:18.566418Z","shell.execute_reply.started":"2026-10-20T00:40:15.737119Z","shell.execute_reply":"2026-10-20T00:40:18.565671Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"Executing original SQL: 0%| | 0/32168 [00:00\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
candidatesexecutesrequote_matchesis_non_empty
0321681.01.00.902
\n"},"metadata":{}},{"name":"stdout","text":"Requoted SQL that still has double-quoted identifiers: 8\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Empty DataFrame\nColumns: [sql, sql_requoted]\nIndex: []","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n
sqlsql_requoted
\n
"},"metadata":{}}],"execution_count":12},{"id":"cell-19","cell_type":"markdown","source":"## 7. Selection\n\nValid candidates are ranked by: references a column with a space (the failure seen in the BIRD benchmark), number of quoted columns (capped at 2), then a random tie-break. The rare reasons are filled first: each entry of `REASON_QUOTAS` takes up to that many rows with the reason, then the rest of `TARGET_SIZE` follows the ranking. Each table is used once, which spreads the selection over as many schemas as possible.","metadata":{}},{"id":"cell-20","cell_type":"code","source":"valid = candidates[candidates[\"executes\"] & candidates[\"requote_matches\"] & candidates[\"is_non_empty\"]].copy()\nvalid[\"n_quoted_columns_capped\"] = valid[\"quoted_identifiers\"].str.len().clip(upper=2)\nvalid[\"random_key\"] = np.random.default_rng(SEED).random(len(valid))\n\nranked = valid.sort_values([\"has_space_column\", \"n_quoted_columns_capped\", \"random_key\"], ascending=[False, False, True])\n\nselected_parts = []\nused_table_ids = set()\nfor reason, quota in REASON_QUOTAS.items():\n pool = ranked[\n ~ranked[\"table_id\"].isin(used_table_ids) & ranked[\"quoting_reasons\"].apply(lambda reasons: reason in reasons)\n ]\n part = pool.groupby(\"table_id\").head(1).head(quota).assign(selection_reason=reason)\n selected_parts.append(part)\n used_table_ids.update(part[\"table_id\"])\n\nremaining = TARGET_SIZE - sum(len(part) for part in selected_parts)\nrest = ranked[~ranked[\"table_id\"].isin(used_table_ids)].groupby(\"table_id\").head(1).head(remaining)\nselected = pd.concat([*selected_parts, rest.assign(selection_reason=\"ranking\")])\nselected[\"schema\"] = selected[\"table_id\"].map(tables[\"ddl\"])\nselected[\"evidence\"] = \"\"\nselected[\"sql_original\"] = selected[\"sql\"]\nselected[\"sql\"] = selected[\"sql_requoted\"]\nselected[\"total_tokens\"] = count_example_tokens(selected[\"schema\"], selected[\"evidence\"], selected[\"question\"], selected[\"sql\"])\nselected = selected[selected[\"total_tokens\"] <= MAX_SEQ_LENGTH]\n\nreferences = [quoted_references(sql, tables.at[table_id, \"quoted_columns\"]) for table_id, sql in zip(selected[\"table_id\"], selected[\"sql\"])]\nselected[\"quote_styles\"] = [styles for _, styles in references]\nprint(f\"{len(valid):,} valid candidates from {valid['table_id'].nunique():,} tables\")\nprint(f\"Selected {len(selected):,} rows from {selected['table_id'].nunique():,} tables (target {TARGET_SIZE:,})\")\ndisplay(selected[\"selection_reason\"].value_counts().rename(\"rows\").to_frame())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:40:18.567414Z","iopub.execute_input":"2026-10-20T00:40:18.567672Z","iopub.status.idle":"2026-10-20T00:40:19.386705Z","shell.execute_reply.started":"2026-10-20T00:40:18.567647Z","shell.execute_reply":"2026-10-20T00:40:19.385482Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"Tokenizing: 0%| | 0/1 [00:00\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
rows
selection_reason
ranking700
leading_digit206
reserved_word94
\n"},"metadata":{}}],"execution_count":13},{"id":"cell-21","cell_type":"markdown","source":"## 8. Selection vs BIRD and Spider\n\nThe quoting table shows how much each dataset exercises quoted columns and with which quote style; the selection should be close to 100% with only `IDENTIFIER_QUOTE`. The size table confirms LLMSQL prompts are short, so the subset costs little training time.","metadata":{}},{"id":"cell-22","cell_type":"code","source":"display(pd.DataFrame([\n quoting_summary(\"BIRD train\", bird[\"quoted_identifiers\"], bird[\"quote_styles\"]),\n quoting_summary(\"Spider train\", spider[\"quoted_identifiers\"], spider[\"quote_styles\"]),\n quoting_summary(\"LLMSQL train\", questions[\"quoted_identifiers\"], questions[\"quote_styles\"]),\n quoting_summary(\"LLMSQL selected\", selected[\"quoted_identifiers\"], selected[\"quote_styles\"]),\n]).round(3))\ndisplay(pd.DataFrame([\n token_summary(\"BIRD train\", bird[\"total_tokens\"].to_numpy()),\n token_summary(\"Spider train\", spider[\"total_tokens\"].to_numpy()),\n token_summary(\"LLMSQL selected\", selected[\"total_tokens\"].to_numpy()),\n]).round(3))\ndisplay(pd.concat({\n \"llmsql_train\": questions[\"aggregation\"].value_counts(normalize=True),\n \"selected\": selected[\"aggregation\"].value_counts(normalize=True),\n}, axis=1).round(3))\ndisplay(selected[[\"table_id\", \"question\", \"sql_original\", \"sql\", \"total_tokens\"]].head(5))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:40:19.389965Z","iopub.execute_input":"2026-10-20T00:40:19.390407Z","iopub.status.idle":"2026-10-20T00:40:19.584863Z","shell.execute_reply.started":"2026-10-20T00:40:19.390375Z","shell.execute_reply":"2026-10-20T00:40:19.583847Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":" dataset rows with_quoted_identifier with_space \\\n0 BIRD train 6597 0.074 0.047 \n1 Spider train 8576 0.000 0.000 \n2 LLMSQL train 56111 0.573 0.514 \n3 LLMSQL selected 1000 1.000 0.777 \n\n with_special_character with_leading_digit with_reserved_word \\\n0 0.019 0.000 0.008 \n1 0.000 0.000 0.000 \n2 0.064 0.015 0.003 \n3 0.071 0.206 0.094 \n\n uses_backtick uses_double_quote uses_bracket \n0 0.992 0.01 0.0 \n1 NaN NaN NaN \n2 0.000 1.00 0.0 \n3 1.000 0.00 0.0 ","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
datasetrowswith_quoted_identifierwith_spacewith_special_characterwith_leading_digitwith_reserved_worduses_backtickuses_double_quoteuses_bracket
0BIRD train65970.0740.0470.0190.0000.0080.9920.010.0
1Spider train85760.0000.0000.0000.0000.000NaNNaNNaN
2LLMSQL train561110.5730.5140.0640.0150.0030.0001.000.0
3LLMSQL selected10001.0000.7770.0710.2060.0941.0000.000.0
\n
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":" dataset rows p50 p75 p95 max over_max_seq_length\n0 BIRD train 6597 824.0 1232.0 7267.00 7428 0.058\n1 Spider train 8576 555.0 795.0 1440.00 2757 0.000\n2 LLMSQL selected 1000 218.0 239.0 300.05 511 0.000","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
datasetrowsp50p75p95maxover_max_seq_length
0BIRD train6597824.01232.07267.0074280.058
1Spider train8576555.0795.01440.0027570.000
2LLMSQL selected1000218.0239.0300.055110.000
\n
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":" llmsql_train selected\naggregation \nNONE 0.720 0.854\nCOUNT 0.091 0.067\nMIN 0.058 0.023\nMAX 0.056 0.028\nAVG 0.039 0.017\nSUM 0.036 0.011","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
llmsql_trainselected
aggregation
NONE0.7200.854
COUNT0.0910.067
MIN0.0580.023
MAX0.0560.028
AVG0.0390.017
SUM0.0360.011
\n
"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":" table_id \\\n51408 2-12334508-4 \n12174 1-2570269-3 \n55008 2-13218356-2 \n29548 2-16772687-6 \n50572 2-18504368-1 \n\n question \\\n51408 What order has the Species Authority of Hydrochaeris hydrochaeris (Linnaeus, 1766)? \n12174 Who are the cast members of episode 3-04? \n55008 What is the Chinese name of the Eliminated player from Singapore in Index F9? \n29548 Where is the player from that wears the number 29? \n50572 Which Effective date has a Where of Illinois, and Affected area codes of 217? \n\n sql_original \\\n51408 SELECT \"Order\" FROM \"2-12334508-4\" WHERE \"Species Authority\" = 'Hydrochaeris hydrochaeris (Linnaeus, 1766)'; \n12174 SELECT \"Cast\" FROM \"1-2570269-3\" WHERE \"Episode #\" = '3-04'; \n55008 SELECT \"Chinese Name\" FROM \"2-13218356-2\" WHERE \"Status\" = 'Eliminated' AND \"Country\" = 'Singapore' AND \"Index\" = 'F9'; \n29548 SELECT \"From\" FROM \"2-16772687-6\" WHERE \"Jersey Number(s)\" = '29'; \n50572 SELECT \"Effective date\" FROM \"2-18504368-1\" WHERE \"Where\" = 'Illinois' AND \"Affected area codes\" = '217'; \n\n sql \\\n51408 SELECT `Order` FROM `2-12334508-4` WHERE `Species Authority` = 'Hydrochaeris hydrochaeris (Linnaeus, 1766)'; \n12174 SELECT `Cast` FROM `1-2570269-3` WHERE `Episode #` = '3-04'; \n55008 SELECT `Chinese Name` FROM `2-13218356-2` WHERE `Status` = 'Eliminated' AND `Country` = 'Singapore' AND `Index` = 'F9'; \n29548 SELECT `From` FROM `2-16772687-6` WHERE `Jersey Number(s)` = '29'; \n50572 SELECT `Effective date` FROM `2-18504368-1` WHERE `Where` = 'Illinois' AND `Affected area codes` = '217'; \n\n total_tokens \n51408 216 \n12174 193 \n55008 205 \n29548 194 \n50572 207 ","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
table_idquestionsql_originalsqltotal_tokens
514082-12334508-4What order has the Species Authority of Hydrochaeris hydrochaeris (Linnaeus, 1766)?SELECT \"Order\" FROM \"2-12334508-4\" WHERE \"Species Authority\" = 'Hydrochaeris hydrochaeris (Linnaeus, 1766)';SELECT `Order` FROM `2-12334508-4` WHERE `Species Authority` = 'Hydrochaeris hydrochaeris (Linnaeus, 1766)';216
121741-2570269-3Who are the cast members of episode 3-04?SELECT \"Cast\" FROM \"1-2570269-3\" WHERE \"Episode #\" = '3-04';SELECT `Cast` FROM `1-2570269-3` WHERE `Episode #` = '3-04';193
550082-13218356-2What is the Chinese name of the Eliminated player from Singapore in Index F9?SELECT \"Chinese Name\" FROM \"2-13218356-2\" WHERE \"Status\" = 'Eliminated' AND \"Country\" = 'Singapore' AND \"Index\" = 'F9';SELECT `Chinese Name` FROM `2-13218356-2` WHERE `Status` = 'Eliminated' AND `Country` = 'Singapore' AND `Index` = 'F9';205
295482-16772687-6Where is the player from that wears the number 29?SELECT \"From\" FROM \"2-16772687-6\" WHERE \"Jersey Number(s)\" = '29';SELECT `From` FROM `2-16772687-6` WHERE `Jersey Number(s)` = '29';194
505722-18504368-1Which Effective date has a Where of Illinois, and Affected area codes of 217?SELECT \"Effective date\" FROM \"2-18504368-1\" WHERE \"Where\" = 'Illinois' AND \"Affected area codes\" = '217';SELECT `Effective date` FROM `2-18504368-1` WHERE `Where` = 'Illinois' AND `Affected area codes` = '217';207
\n
"},"metadata":{}}],"execution_count":14},{"id":"cell-23","cell_type":"code","source":"def plot_reason_shares(frames):\n reason_shares = pd.DataFrame({\n label: [\n frame[\"quoted_identifiers\"].apply(lambda names: any(quoting_reason(name) == reason for name in names)).mean()\n for reason in QUOTING_REASONS\n ]\n for label, (frame, _) in frames.items()\n }, index=QUOTING_REASONS)\n fig, axis = plt.subplots(figsize=(11, 4))\n reason_shares.plot.bar(ax=axis, color=[color for _, color in frames.values()], width=0.8)\n axis.set(title=\"Share of rows referencing a name that needs quotes, by reason\", xlabel=\"\", ylabel=\"share of rows\")\n axis.tick_params(axis=\"x\", rotation=0)\n plt.show()\n return reason_shares\n\n\ndisplay(plot_reason_shares({\n \"BIRD train\": (bird, DATASET_COLORS[\"BIRD\"]),\n \"Spider train\": (spider, DATASET_COLORS[\"Spider\"]),\n \"LLMSQL selected\": (selected, DATASET_COLORS[\"LLMSQL\"]),\n}).round(3))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:40:19.585887Z","iopub.execute_input":"2026-10-20T00:40:19.586211Z","iopub.status.idle":"2026-10-20T00:40:19.917320Z","shell.execute_reply.started":"2026-10-20T00:40:19.586183Z","shell.execute_reply":"2026-10-20T00:40:19.915897Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"
","image/png":"iVBORw0KGgoAAAANSUhEUgAAA5sAAAF2CAYAAADgLoxlAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAcHJJREFUeJzt3XlcTekfB/DPLbqtt0UrUhQqWyM0mMTIZIuMfZnIPjPWmMEP2YZsQwbDjBnLGMa+/exExtJgZJesCaMSKhWl7vP7Y16dn6vtXm6u+Lxfr/t6dZ/zPOf5nnPPObfvPec8RyaEECAiIiIiIiLSIj1dB0BERERERETvHyabREREREREpHVMNomIiIiIiEjrmGwSERERERGR1jHZJCIiIiIiIq1jsklERERERERax2STiIiIiIiItI7JJhEREREREWkdk00iIiIiIiLSOiabRKWETCbDkCFDdB2GViUmJqJTp04oV64cZDIZwsPDdR3SOyU9PR39+/eHvb09ZDIZRowYoeuQ3sjkyZMhk8l0HQYVIe8zSk5O1nUo74y4uDjIZDKsXLlS16G8V1auXAmZTIa///5b16EQUQliskmkYxcvXkSnTp3g5OQEQ0NDVKhQAS1atMDChQt1HVqJGzlyJPbt24dx48Zh9erVaNmypa5DeqfMmDEDK1euxJdffonVq1fjiy++0HVI9J6YMWMGtm3b9lb6+vHHH5movQNOnDiByZMnIyUlRdehENEHhMkmkQ6dOHEC9erVw/nz5zFgwAAsWrQI/fv3h56eHhYsWKDr8ErcoUOH0L59e4wePRq9evWCm5ubrkN6pxw6dAgff/wxJk2ahF69esHLy0vXIb2RCRMm4NmzZ7oOg8Bk80N04sQJTJkyhckmEb1VZXQdANGHbPr06TA3N8fp06dhYWGhMi0pKemtx5ORkQETE5O31l9SUlK+5VZXTk4OlEolDAwMtBtUCRFC4Pnz5zAyMlK7TVJSEjw8PLQWg1KpRHZ2NgwNDbU2T02UKVMGZcrwa4eIdCszMxPGxsa6DoPog8Azm0Q6dPPmTdSoUaPAhMvW1rbANtu2bUPNmjUhl8tRo0YN7N27V2X6nTt38NVXX6F69eowMjJCuXLl0LlzZ8TFxanUy7tf5siRI/jqq69ga2uLihUrStP37NkDHx8fmJiYwMzMDG3atMHly5fVWq5bt26hc+fOsLKygrGxMT7++GPs2rUrX99CCCxevBgymazIe/ny7pmaO3cuwsPD4eLiArlcjitXrgD49wxgXqwWFhZo3749YmJipPYXLlyATCbDjh07pLIzZ85AJpOhbt26Kn21atUK3t7e0vu///4b/v7+sLa2hpGRESpXroy+ffsWuw6cnZ3Rtm1b7Nu3D/Xq1YORkRF++uknAEBKSgpGjBgBR0dHyOVyuLq6YtasWVAqlQCAyMhIyGQy3L59G7t27ZLWT95nmJWVhUmTJsHV1RVyuRyOjo749ttvkZWVpRJD3n2+a9asQY0aNSCXy6Xt5f79++jbty/s7OykbWn58uUq7fPi2LBhA6ZPn46KFSvC0NAQzZs3x40bN/It88mTJ9G6dWtYWlrCxMQEtWvXVjlDX9A9m3kxFrdd58VTr149GBoawsXFBT/99JPa94EePXoUnTt3RqVKlaR1NnLkSLXOtOZtr8ePH0dISAhsbGxgYmKCDh064OHDhyp1t2/fjjZt2qB8+fKQy+VwcXHBtGnTkJubq1KvadOmqFmzJi5cuABfX18YGxvD1dUVmzZtAgAcOXIE3t7eMDIyQvXq1XHw4MF8canzGRZEJpMhIyMDq1atkratPn36qNRJSUlBnz59YGFhAXNzcwQHByMzM1OlzooVK/Dpp5/C1tYWcrkcHh4eWLJkiUodZ2dnXL58GUeOHJH6atq0aaGxvbyv//zzz9K+Xr9+fZw+fTpf/atXr6JTp06wsrKCoaEh6tWrp7Kfv7w8Re1zry63ubk5LCws0Lt37wLPBCYkJCA4OBgVK1aEXC6Hg4MD2rdvn+84W5C8bd3Q0BA1a9bE1q1b0adPHzg7O0t18va9yMjIAtfPq2eKizsGTp48Gd988w0AoHLlyvmOKQDw+++/w8vLC0ZGRrCyskK3bt1w9+5dlX6uX7+Ojh07wt7eHoaGhqhYsSK6deuG1NTUYpe7MJmZmRg0aBDKlSsHhUKBoKAgPHnyRJreu3dvWFtb48WLF/nafvbZZ6hevXqR88/b186cOYMmTZrA2NgY//nPfwCofyxVZ1sH1Pu+yMjIwKhRo6RtsXr16pg7dy6EECr1NDk2Er3L+BMzkQ45OTkhKioKly5dQs2aNYutf+zYMWzZsgVfffUVzMzM8MMPP6Bjx46Ij49HuXLlAACnT5/GiRMn0K1bN1SsWBFxcXFYsmQJmjZtiitXruT7Nferr76CjY0NQkNDkZGRAQBYvXo1evfuDX9/f8yaNQuZmZlYsmQJPvnkE5w9e1bln6JXJSYmolGjRsjMzMSwYcNQrlw5rFq1Cu3atcOmTZvQoUMHNGnSRLoHsUWLFggKClJrfa1YsQLPnz/HwIEDIZfLYWVlhYMHD6JVq1aoUqUKJk+ejGfPnmHhwoVo3LgxoqOj4ezsjJo1a8LCwgJ//vkn2rVrB+Df5ENPTw/nz59HWloaFAoFlEolTpw4gYEDBwL498ziZ599BhsbG4wdOxYWFhaIi4vDli1b1Io3NjYW3bt3x6BBgzBgwABUr14dmZmZ8PX1xf379zFo0CBUqlQJJ06cwLhx4/DgwQOEh4fD3d0dq1evxsiRI1GxYkWMGjUKAGBjYwOlUol27drh2LFjGDhwINzd3XHx4kXMnz8f165dy3dp5KFDh7BhwwYMGTIE1tbWcHZ2RmJiIj7++GPpnxkbGxvs2bMH/fr1Q1paWr6BiGbOnAk9PT2MHj0aqampmD17Nnr27ImTJ09KdQ4cOIC2bdvCwcEBw4cPh729PWJiYrBz504MHz68yPWkznZ99uxZtGzZEg4ODpgyZQpyc3MxdepU2NjYqPVZbNy4EZmZmfjyyy9Rrlw5nDp1CgsXLsS9e/ewceNGteYxdOhQWFpaYtKkSYiLi0N4eDiGDBmC9evXS3VWrlwJU1NThISEwNTUFIcOHUJoaCjS0tIwZ84clfk9efIEbdu2Rbdu3dC5c2csWbIE3bp1w5o1azBixAgMHjwYPXr0wJw5c9CpUyfcvXsXZmZmAKDxZ/iy1atXo3///mjQoIG0rbu4uKjU6dKlCypXroywsDBER0fjl19+ga2tLWbNmiXVWbJkCWrUqIF27dqhTJky+O9//4uvvvoKSqUSX3/9NQAgPDwcQ4cOhampKcaPHw8AsLOzK3Zdr127Fk+fPsWgQYMgk8kwe/ZsfP7557h16xbKli0LALh8+TIaN26MChUqYOzYsTAxMcGGDRsQGBiIzZs3o0OHDgCg1j4H/Hv1Qfv27XHs2DEMHjwY7u7u2Lp1K3r37p0vvo4dO+Ly5csYOnQonJ2dkZSUhAMHDiA+Pr7I4+P+/fvRsWNHeHh4ICwsDI8ePZKS1telzjHw888/x7Vr1/DHH39g/vz5sLa2BgBp/5k+fTomTpyILl26oH///nj48CEWLlyIJk2a4OzZs7CwsEB2djb8/f2RlZWFoUOHwt7eHvfv38fOnTuRkpICc3Pz14p/yJAhsLCwwOTJkxEbG4slS5bgzp07UsL9xRdf4LfffsO+ffvQtm1bqV1CQgIOHTqESZMmFdvHo0eP0KpVK3Tr1g29evWCnZ2dRsdSdbZ1db4vhBBo164dDh8+jH79+sHT0xP79u3DN998g/v372P+/PkqcatzbCR65wki0pn9+/cLfX19oa+vLxo2bCi+/fZbsW/fPpGdnZ2vLgBhYGAgbty4IZWdP39eABALFy6UyjIzM/O1jYqKEgDEb7/9JpWtWLFCABCffPKJyMnJkcqfPn0qLCwsxIABA1TmkZCQIMzNzfOVv2rEiBECgDh69KjKPCtXriycnZ1Fbm6uyjJ9/fXXRc5PCCFu374tAAiFQiGSkpJUpnl6egpbW1vx6NEjqez8+fNCT09PBAUFSWVt2rQRDRo0kN5//vnn4vPPPxf6+vpiz549QgghoqOjBQCxfft2IYQQW7duFQDE6dOni43xVU5OTgKA2Lt3r0r5tGnThImJibh27ZpK+dixY4W+vr6Ij49XmUebNm1U6q1evVro6emprF8hhFi6dKkAII4fPy6VARB6enri8uXLKnX79esnHBwcRHJyskp5t27dhLm5ubQNHT58WAAQ7u7uIisrS6q3YMECAUBcvHhRCCFETk6OqFy5snBychJPnjxRmadSqZT+njRpknj1a0fd7TogIEAYGxuL+/fvS2XXr18XZcqUyTfPghS0X4SFhQmZTCbu3LlTZNu8fcXPz09leUaOHCn09fVFSkpKkf0MGjRIGBsbi+fPn0tlvr6+AoBYu3atVHb16lXpM/vrr7+k8n379gkAYsWKFVKZup9hYUxMTETv3r3zled9Rn379lUp79ChgyhXrpxKWUF9+Pv7iypVqqiU1ahRQ/j6+hYZT568fb1cuXLi8ePHUvn27dsFAPHf//5XKmvevLmoVauWynpVKpWiUaNGomrVqlKZuvvctm3bBAAxe/ZsqU5OTo7w8fFRWf9PnjwRAMScOXPUWqaXeXp6CgcHB5VtZv/+/QKAcHJyksry9r3Dhw8XuH5e3hbUPQbOmTNHABC3b99WmWdcXJzQ19cX06dPVym/ePGiKFOmjFR+9uxZAUBs3LhR4+UuSN5+5eXlpfKdN3v2bJXjcG5urqhYsaLo2rWrSvt58+YJmUwmbt26VWQ/efva0qVLVco1OZaqs62r832Rt4199913KuWdOnUSMplM5Tio7rGR6F3Hy2iJdKhFixaIiopCu3btcP78ecyePRv+/v6oUKFCgZeC+fn5qZyBqF27NhQKBW7duiWVvXxP4IsXL/Do0SO4urrCwsIC0dHR+eY5YMAA6OvrS+8PHDiAlJQUdO/eHcnJydJLX18f3t7eOHz4cJHLtHv3bjRo0ACffPKJVGZqaoqBAwciLi5OuvT1dXTs2FHlTNaDBw9w7tw59OnTB1ZWVlJ57dq10aJFC+zevVsq8/HxQXR0tHT29tixY2jdujU8PT1x9OhRAP+e7ZTJZFLseZc379y5s8BLuIpTuXJl+Pv7q5Rt3LgRPj4+sLS0VFm/fn5+yM3NxZ9//lnkPDdu3Ah3d3e4ubmptP/0008BIN/n4+vrq3LfpxACmzdvRkBAAIQQKvPw9/dHampqvu0kODhY5d5YHx8fAJC2u7Nnz+L27dsYMWJEvkvC1bnEtbjtOjc3FwcPHkRgYCDKly8v1XN1dUWrVq2KnT+gul9kZGQgOTkZjRo1ghACZ8+eVWseAwcOVFkeHx8f5Obm4s6dOwX28/TpUyQnJ8PHxweZmZm4evWqyvxMTU3RrVs36X316tVhYWEBd3d3lUu58/7OWx+v8xlqavDgwSrvfXx88OjRI6SlpRW4rKmpqUhOToavry9u3br1RpdVAkDXrl1haWmp0j/w/3Xw+PFjHDp0CF26dJHWc3JyMh49egR/f39cv34d9+/fB6D+Prd7926UKVMGX375pdSvvr4+hg4dqhKbkZERDAwMEBkZqXK5Z3Hyjle9e/dWOQvYokWL1743W5NjYGG2bNkCpVKJLl26qKwfe3t7VK1aVTqm5MW8b9++fJdUv4mBAwdKZ6sB4Msvv0SZMmWk2PX09NCzZ0/s2LEDT58+leqtWbMGjRo1QuXKlYvtQy6XIzg4WKVMk2OpOtu6Ot8Xu3fvhr6+PoYNG6ZSPmrUKAghsGfPHpVydb7zid51vIyWSMfq16+PLVu2IDs7G+fPn8fWrVsxf/58dOrUCefOnVP5J6RSpUr52ltaWqr8w/Ps2TOEhYVhxYoVuH//vsp9IAX9A/jqF/X169cBQPrCfZVCoShyee7cuaPyj3Ied3d3abo6lwwX5NVY8/7JL+ieHXd3d+zbt08a9MjHxwc5OTmIioqCo6MjkpKS4OPjg8uXL6skmx4eHtI/bb6+vujYsSOmTJmC+fPno2nTpggMDESPHj0gl8s1jhf4d/1euHCh0Ms/ixsY6vr164iJiVG7/asxPHz4ECkpKfj555/x888/qzWPV7e7vCQgb7u7efMmALz251rcdp2UlIRnz57B1dU1X72CygoSHx+P0NBQ7NixI1+CoG5iVNx6AP69tHPChAk4dOiQSmJWUD8VK1bMl4ybm5vD0dExX9nL/bzOZ6ipopY17xhw/PhxTJo0CVFRUfmSj9TU1Ne+rLK4/gHgxo0bEEJg4sSJmDhxYoHzSEpKQoUKFdTe5+7cuQMHBweYmpqqTH/1+CKXyzFr1iyMGjUKdnZ2+Pjjj9G2bVsEBQXB3t6+0GXKO15VrVo137Tq1au/1g8EmhwDC3P9+nUIIQqMC4CUCFauXBkhISGYN28e1qxZAx8fH7Rr1w69evV6o8/61X5NTU3h4OCgcj9pUFAQZs2aha1btyIoKAixsbE4c+YMli5dqlYfFSpUyDeYnCbHUnW2dXW+L+7cuYPy5ctLl8Pnefn78WXqfOcTveuYbBK9IwwMDFC/fn3Ur18f1apVQ3BwMDZu3KhyP8rLZyBf9nJCOXToUKxYsQIjRoxAw4YNYW5uDplMhm7duuUbDANAvtFR8+qsXr26wH+cdDmaqCYjub4qb2CZP//8E5UqVYKtrS2qVasGHx8f/Pjjj8jKysLRo0el+7yAf8/Kbdq0CX/99Rf++9//Yt++fejbty++//57/PXXX/n+KVUnXqVSiRYtWuDbb78tsE21atWKnKdSqUStWrUwb968Aqe/mqgU9vn26tWrwHvRgH9/PX+ZOtvdmyjp+efm5qJFixZ4/PgxxowZAzc3N5iYmOD+/fvo06dPgfvF68SZkpICX19fKBQKTJ06FS4uLjA0NER0dDTGjBmTr5/C5ldcP6/zGWqquBhu3ryJ5s2bw83NDfPmzYOjoyMMDAywe/duzJ8/X+11+rr9581/9OjR+a4eyJP3Q8Sb7nMFGTFiBAICArBt2zbs27cPEydORFhYGA4dOoSPPvpI4/m9qrArAl4daEoblEolZDIZ9uzZU+B6f/k49/3336NPnz7Yvn079u/fj2HDhiEsLAx//fXXG913WhwPDw94eXnh999/R1BQEH7//XcYGBigS5cuarUv7FiszrFU3W39Tb8vClLSx0ait4HJJtE7qF69egD+vURKU5s2bULv3r3x/fffS2XPnz9X+9lqeZfs2Nraws/PT+P+nZycEBsbm6887xJCJycnjedZVF8ACu3P2tpa+kXfwMAADRo0wNGjR1GpUiXpsjwfHx9kZWVhzZo1SExMRJMmTfLN6+OPP8bHH3+M6dOnY+3atejZsyfWrVuH/v37axyzi4sL0tPTX2vd5rU/f/48mjdvrtYlqq+ysbGBmZkZcnNzXzuGgmICgEuXLmltni+ztbWFoaFhgSPgFlT2qosXL+LatWtYtWqVymBUBw4c0GqckZGRePToEbZs2aKyHd2+fVur/WjjM3ydbedl//3vf5GVlYUdO3aonH0p6DL7N+2rIFWqVAHw71m34taBuvuck5MTIiIikJ6erpIYFHR8yZvvqFGjMGrUKFy/fh2enp74/vvv8fvvvxc6f+D/V4+87NU+8s7kvnrcfvXMlybHwMI+BxcXFwghULlyZbUS71q1aqFWrVqYMGECTpw4gcaNG2Pp0qX47rvvim1bkOvXr6NZs2bS+/T0dDx48ACtW7dWqRcUFISQkBA8ePAAa9euRZs2bVQutdaUusdSTbZ1oOjvCycnJxw8eBBPnz5VObtZEt+PRO8K3rNJpEOHDx8u8BfKvHtVihvSvSD6+vr55rlw4UK1fxH39/eHQqHAjBkzCrzv5NVHPbyqdevWOHXqFKKioqSyjIwM/Pzzz3B2dtbqcyMdHBzg6emJVatWqfxTdunSJezfvz/fPys+Pj44efIkDh8+LCWb1tbWcHd3l0bZzCsH/r1k79V16enpCQD5hsZXV5cuXRAVFYV9+/blm5aSkoKcnJxi29+/fx/Lli3LN+3Zs2fSPamF0dfXR8eOHbF582ZcunQp3/TiPt+C1K1bF5UrV0Z4eHi+f4618Qu8vr4+/Pz8sG3bNvzzzz9S+Y0bN/Ld41RY+1djEUKoPJZFGwrqJzs7Gz/++KPW+3nTz9DExETtH6AKiwFAvsv0V6xYofW+CmJra4umTZvip59+KvBHuZfXgbr7XOvWrZGTk6PySIvc3FwsXLhQpU1mZiaeP3+uUubi4gIzM7MijwsvH69evqT6wIED+e5ld3Jygr6+fr57uF/dljQ5BuYlna9+Fp9//jn09fUxZcqUfPurEAKPHj0CAKSlpeU7PtWqVQt6enqvfTwEgJ9//lnlu2bJkiXIycnJdz929+7dIZPJMHz4cNy6dQu9evV67T4B9Y+l6m7r6nxftG7dGrm5uVi0aJFKvfnz50Mmk6l9DzpRacIzm0Q6NHToUGRmZqJDhw5wc3NDdnY2Tpw4gfXr18PZ2TnfgAbqaNu2LVavXg1zc3N4eHggKioKBw8eVHuYdIVCgSVLluCLL75A3bp10a1bN9jY2CA+Ph67du1C48aN831Rvmzs2LH4448/0KpVKwwbNgxWVlZYtWoVbt++jc2bN0NPT7u/cc2ZMwetWrVCw4YN0a9fP2nYf3Nzc0yePFmlro+PD6ZPn467d++qJJVNmjTBTz/9BGdnZ5VLwVatWoUff/wRHTp0gIuLC54+fYply5ZBoVDkS2TV9c0332DHjh1o27Yt+vTpAy8vL2RkZODixYvYtGkT4uLipMcSFOSLL77Ahg0bMHjwYBw+fBiNGzdGbm4url69ig0bNkjP9SzKzJkzcfjwYXh7e2PAgAHw8PDA48ePER0djYMHD+Lx48caLZOenh6WLFmCgIAAeHp6Ijg4GA4ODrh69SouX75c4D/5mpo8eTL279+Pxo0b48svv5T+YatZsybOnTtXZFs3Nze4uLhg9OjRuH//PhQKBTZv3qz1+54aNWoES0tL9O7dG8OGDYNMJsPq1atL5JK3N/0Mvby8cPDgQcybNw/ly5dH5cqVC7zXujCfffYZDAwMEBAQgEGDBiE9PR3Lli2Dra1tvuTPy8sLS5YswXfffQdXV1fY2toWek+4JhYvXoxPPvkEtWrVwoABA1ClShUkJiYiKioK9+7dw/nz5wGov88FBASgcePGGDt2LOLi4uDh4YEtW7bku9f22rVraN68Obp06QIPDw+UKVMGW7duRWJiosqATwUJCwtDmzZt8Mknn6Bv3754/PgxFi5ciBo1aiA9PV2qZ25ujs6dO2PhwoWQyWRwcXHBzp07C7wXV91joJeXFwBg/Pjx6NatG8qWLYuAgAC4uLjgu+++w7hx4xAXF4fAwECYmZnh9u3b2Lp1KwYOHIjRo0fj0KFDGDJkCDp37oxq1aohJycHq1evln78yDN58mRMmTIFhw8fLvKZqnmys7Ol9RkbG4sff/wRn3zyifSYqjw2NjZo2bIlNm7cCAsLC7Rp06bYeRdF3WOputu6Ot8XAQEBaNasGcaPH4+4uDjUqVMH+/fvx/bt2zFixIh8jyAiei+8hRFviagQe/bsEX379hVubm7C1NRUGBgYCFdXVzF06FCRmJioUheFPCbEyclJ5REGT548EcHBwcLa2lqYmpoKf39/cfXq1Xz18oadL2yY9sOHDwt/f39hbm4uDA0NhYuLi+jTp4/4+++/i12umzdvik6dOgkLCwthaGgoGjRoIHbu3JmvXmHL9Kq84f4Le9TAwYMHRePGjYWRkZFQKBQiICBAXLlyJV+9tLQ0oa+vL8zMzFQe9/L7778LAOKLL75QqR8dHS26d+8uKlWqJORyubC1tRVt27ZVax0U9NiSPE+fPhXjxo0Trq6uwsDAQFhbW4tGjRqJuXPnqjwCoLB5ZGdni1mzZokaNWoIuVwuLC0thZeXl5gyZYpITU2V6hW1fhMTE8XXX38tHB0dRdmyZYW9vb1o3ry5+Pnnn6U6eY9fePVRBwU9fkEIIY4dOyZatGghzMzMhImJiahdu7bKEP2FPfpEne1aCCEiIiLERx99JAwMDISLi4v45ZdfxKhRo4ShoWGBy/iyK1euCD8/P2Fqaiqsra3FgAEDpMcIvLocrypsXyno8RTHjx8XH3/8sTAyMhLly5eXHmf0aj1fX19Ro0aNApe7oM+8oPWkzmdYmKtXr4omTZoIIyMjAUBa13mf0cOHDwtcBy8/NmPHjh2idu3awtDQUDg7O4tZs2aJ5cuX56uXkJAg2rRpI8zMzASAIh+DUtS+DkBMmjRJpezmzZsiKChI2Nvbi7Jly4oKFSqItm3bik2bNqnUU3efe/Tokfjiiy+EQqEQ5ubm4osvvpAe+ZG3nSQnJ4uvv/5auLm5CRMTE2Fubi68vb3Fhg0bCl/hL9m8ebNwd3cXcrlceHh4iC1btojevXurPPpECCEePnwoOnbsKIyNjYWlpaUYNGiQuHTpUoHbrLrHwGnTpokKFSoIPT29fJ/T5s2bxSeffCJMTEyEiYmJcHNzE19//bWIjY0VQghx69Yt0bdvX+Hi4iIMDQ2FlZWVaNasmTh48KBKH6NGjRIymUzExMQUuR7ytqkjR46IgQMHCktLS2Fqaip69uyp8hiXl23YsEEAEAMHDixy3i8rbF8TQv1jqTrburrfF0+fPhUjR44U5cuXF2XLlhVVq1YVc+bMUXmskhCaHRuJ3mUyIXiXMRERlU6BgYG4fPlygffBEZUWffr0QWRkpMoIrKVVgwYN4OTkhI0bN2p93tu3b0dgYCD+/PNPlatTiOjdxXs2iYioVHj27JnK++vXr2P37t1qXapHRCUvLS0N58+fx9SpU0tk/suWLUOVKlVUnuNMRO823rNJRESlQpUqVdCnTx9UqVIFd+7cwZIlS2BgYFDoIy2I6O1SKBRvNFhQYdatW4cLFy5g165dWLBgQYmMcExEJYPJJhERlQotW7bEH3/8gYSEBMjlcjRs2BAzZswo9GH0RPR+6N69O0xNTdGvXz989dVXug6HiDTAezaJiIiIiIhI63jPJhEREREREWkdk00iIiIiIiLSug/unk2lUol//vkHZmZmvMGciIiIiIhIQ0IIPH36FOXLl4eeXuHnLz+4ZPOff/6Bo6OjrsMgIiIiIiIq1e7evYuKFSsWOv2DSzbNzMwA/LtiFAqFjqMhIiIiIiIqXdLS0uDo6CjlVoXRebK5ePFizJkzBwkJCahTpw4WLlyIBg0aFFo/PDwcS5YsQXx8PKytrdGpUyeEhYXB0NBQrf7yLp1VKBRMNomIiIiIiF5Tcbcl6nSAoPXr1yMkJASTJk1CdHQ06tSpA39/fyQlJRVYf+3atRg7diwmTZqEmJgY/Prrr1i/fj3+85//vOXIiYiIiIiIqCg6TTbnzZuHAQMGIDg4GB4eHli6dCmMjY2xfPnyAuufOHECjRs3Ro8ePeDs7IzPPvsM3bt3x6lTp95y5ERERERERFQUnSWb2dnZOHPmDPz8/P4fjJ4e/Pz8EBUVVWCbRo0a4cyZM1JyeevWLezevRutW7cutJ+srCykpaWpvIiIiIiIiKhk6eyezeTkZOTm5sLOzk6l3M7ODlevXi2wTY8ePZCcnIxPPvkEQgjk5ORg8ODBRV5GGxYWhilTpmg1diIiIiIibcrNzcWLFy90HQYRAKBs2bLQ19d/4/nofIAgTURGRmLGjBn48ccf4e3tjRs3bmD48OGYNm0aJk6cWGCbcePGISQkRHqfN3ISEREREZGuCSGQkJCAlJQUXYdCpMLCwgL29vbFDgJUFJ0lm9bW1tDX10diYqJKeWJiIuzt7QtsM3HiRHzxxRfo378/AKBWrVrIyMjAwIEDMX78+AIfKCqXyyGXy7W/AEREREREbygv0bS1tYWxsfEb/WNPpA1CCGRmZkqDtjo4OLz2vHSWbBoYGMDLywsREREIDAwEACiVSkRERGDIkCEFtsnMzMyXUOad3hVClGi8RERERETalJubKyWa5cqV03U4RBIjIyMAQFJSEmxtbV/7klqdXkYbEhKC3r17o169emjQoAHCw8ORkZGB4OBgAEBQUBAqVKiAsLAwAEBAQADmzZuHjz76SLqMduLEiQgICNDKNcVERERERG9L3j2axsbGOo6EKL+87fLFixelM9ns2rUrHj58iNDQUCQkJMDT0xN79+6VBg2Kj49XOZM5YcIEyGQyTJgwAffv34eNjQ0CAgIwffp0XS0CEREREdEb4aWz9C7SxnYpEx/Y9adpaWkwNzdHamoqFAqFrsMhIiIiog/U8+fPcfv2bVSuXBmGhoa6DodIRVHbp7o5lc6es0lERERERPQumzx5Mjw9PXUdRqlVqh59QkRERET0IagxLuat9XU5zF3jNn369MGqVauk91ZWVqhfvz5mz56N2rVrS+UymQxbt26VBgR9+dJMMzMzVK9eHRMmTED79u2l8pUrV0pjuOjp6UGhUKBatWpo06YNhg8fDnNz80LjWrlyJUaMGKG1R8mMHj0aQ4cO1cq8PkRMNolKWMVt43UdglbdC+Q90kRERAS0bNkSK1asAPDvI1wmTJiAtm3bIj4+vsh2K1asQMuWLZGWloYff/wRnTp1QnR0NGrVqiXVUSgUiI2NhRACKSkpOHHiBMLCwrBixQocP34c5cuXf6PYs7OzYWBgUGw9U1NTmJqavlFfHzJeRktERERERBqTy+Wwt7eHvb09PD09MXbsWNy9excPHz4ssp2FhQXs7e1RrVo1TJs2DTk5OTh8+LBKHZlMBnt7ezg4OMDd3R39+vXDiRMnkJ6ejm+//bbA+UZGRiI4OBipqamQyWSQyWSYPHkyAMDZ2RnTpk1DUFAQFAoFBg4cCAAYM2YMqlWrBmNjY1SpUgUTJ06URgkG8l9G26dPHwQGBmLu3LlwcHBAuXLl8PXXX6u0of/jmU0iIiIiInoj6enp+P333+Hq6qr2M0NzcnLw66+/AoBaZxltbW3Rs2dPLF++HLm5ufkex9GoUSOEh4cjNDQUsbGxAKByVnLu3LkIDQ3FpEmTpDIzMzOsXLkS5cuXx8WLFzFgwACYmZkVmtACwOHDh+Hg4IDDhw/jxo0b6Nq1Kzw9PTFgwAC1lvtDwmSTiIiIiIg0tnPnTimZy8jIgIODA3bu3Kny6MKCdO/eHfr6+nj27BmUSiWcnZ3RpUsXtfp0c3PD06dP8ejRI9ja2qpMMzAwgLm5uXRW9FWffvopRo0apVI2YcIE6W9nZ2eMHj0a69atKzLZtLS0xKJFi6Cvrw83Nze0adMGERERTDYLwMtoiYiIiIhIY82aNcO5c+dw7tw5nDp1Cv7+/mjVqhXu3LlTZLv58+fj3Llz2LNnDzw8PPDLL7/AyspKrT7zntr4Os+ArFevXr6y9evXo3HjxrC3t4epqSkmTJhQ7D2nNWrUUDmr6uDggKSkJI3j+RAw2SQiIiIiIo2ZmJjA1dUVrq6uqF+/Pn755RdkZGRg2bJlRbazt7eHq6srPvvsM6xYsQJdu3ZVO1mLiYmBQqFQ+1LdV+N9WVRUFHr27InWrVtj586dOHv2LMaPH4/s7Owi51O2bFmV9zKZDEqlUuN4PgRMNomIiIiI6I3JZDLo6enh2bNnardp0KABvLy8MH168aPdJyUlYe3atQgMDCz0Ul0DAwPk5uaq1feJEyfg5OSE8ePHo169eqhatWqxZ2VJM0w2iYiIiIhIY1lZWUhISEBCQgJiYmIwdOhQpKenIyAgQKP5jBgxAj/99BPu378vlQkhkJCQgAcPHiAmJgbLly9Ho0aNYG5ujpkzZxY6L2dnZ6SnpyMiIgLJycnIzMwstG7VqlURHx+PdevW4ebNm/jhhx+wdetWjWKnojHZJCIiIiIije3duxcODg5wcHCAt7c3Tp8+jY0bN6Jp06Yazadly5aoXLmyytnNtLQ0ODg4oEKFCmjYsCF++ukn9O7dG2fPnoWDg0Oh82rUqBEGDx6Mrl27wsbGBrNnzy60brt27TBy5EgMGTIEnp6eOHHiBCZOnKhR7FQ0mci7y/YDkZaWBnNzc6SmpkKhUOg6HPoAVNw2XtchaNW9wOIvcyEiIqLiPX/+HLdv30blypVhaGio63CIVBS1faqbU/HMJhEREREREWkdk00iIiIiIiLSOiabREREREREpHVMNomIiIiIiEjrmGwSERERERGR1jHZJCIiIiIiIq1jsklERERERERax2STiIiIiIiItI7JJhEREREREWkdk00iIiIiInrr4uLiIJPJcO7cuULrREZGQiaTISUl5a3F9bqcnZ0RHh6u6zDeKWV0HQAREREREalK/rL+W+vLeslpjds8fPgQoaGh2LVrFxITE2FpaYk6deogNDQUjRs3Vmsejo6OePDgAaytrTXuXxuaNm0KT09PrSWIp0+fhomJiVbm9b5gsklERERERBrp2LEjsrOzsWrVKlSpUgWJiYmIiIjAo0eP1J6Hvr4+7O3tSzDKf2VnZ8PAwOC12gohkJubizJlik+bbGxsXquP9xkvoyUiIiIiIrWlpKTg6NGjmDVrFpo1awYnJyc0aNAA48aNQ7t27aR6MpkMS5YsQatWrWBkZIQqVapg06ZN0vSCLqPdvXs3qlWrBiMjIzRr1gxxcXH5+j927Bh8fHxgZGQER0dHDBs2DBkZGdJ0Z2dnTJs2DUFBQVAoFBg4cGC+efTp0wdHjhzBggULIJPJIJPJEBcXJ122u2fPHnh5eUEul+PYsWO4efMm2rdvDzs7O5iamqJ+/fo4ePCgyjxfvYxWJpPhl19+QYcOHWBsbIyqVatix44dr7HGSy8mm0REREREpDZTU1OYmppi27ZtyMrKKrLuxIkT0bFjR5w/fx49e/ZEt27dEBMTU2Ddu3fv4vPPP0dAQADOnTuH/v37Y+zYsSp1bt68iZYtW6Jjx464cOEC1q9fj2PHjmHIkCEq9ebOnYs6derg7NmzmDhxYr6+FixYgIYNG2LAgAF48OABHjx4AEdHR2n62LFjMXPmTMTExKB27dpIT09H69atERERgbNnz6Jly5YICAhAfHx8kcs/ZcoUdOnSBRcuXEDr1q3Rs2dPPH78uMg27xMmm0REREREpLYyZcpg5cqVWLVqFSwsLNC4cWP85z//wYULF/LV7dy5M/r3749q1aph2rRpqFevHhYuXFjgfJcsWQIXFxd8//33qF69Onr27Ik+ffqo1AkLC0PPnj0xYsQIVK1aFY0aNcIPP/yA3377Dc+fP5fqffrppxg1ahRcXFzg4uKSry9zc3MYGBjA2NgY9vb2sLe3h76+vjR96tSpaNGiBVxcXGBlZYU6depg0KBBqFmzJqpWrYpp06bBxcWl2DOVffr0Qffu3eHq6ooZM2YgPT0dp06dKrLN++SdSDYXL14MZ2dnGBoawtvbu8gPoGnTptKp7pdfbdq0eYsRExERERF9uDp27Ih//vkHO3bsQMuWLREZGYm6deti5cqVKvUaNmyY731hZzZjYmLg7e1dZPvz589j5cqV0tlVU1NT+Pv7Q6lU4vbt21K9evXqvcHS5W+fnp6O0aNHw93dHRYWFjA1NUVMTEyxZzZr164t/W1iYgKFQoGkpKQ3iq000fkAQevXr0dISAiWLl0Kb29vhIeHw9/fH7GxsbC1tc1Xf8uWLcjOzpbeP3r0CHXq1EHnzp3fZthERERERB80Q0NDtGjRAi1atMDEiRPRv39/TJo0Kd/ZSG1KT0/HoEGDMGzYsHzTKlWqJP39pqPCvtp+9OjROHDgAObOnQtXV1cYGRmhU6dOKnlJQcqWLavyXiaTQalUvlFspYnOz2zOmzcPAwYMQHBwMDw8PLB06VIYGxtj+fLlBda3srKSTnXb29vjwIEDMDY2ZrJJRERERKRDHh4eKgP1AMBff/2V7727u3uB7d3d3fNd4fhq+7p16+LKlStwdXXN99J0xFkDAwPk5uaqVff48ePo06cPOnTogFq1asHe3r7AwYtIlU6TzezsbJw5cwZ+fn5SmZ6eHvz8/BAVFaXWPH799Vd069at0F8vsrKykJaWpvIiIiIiIqLX8+jRI3z66af4/fffceHCBdy+fRsbN27E7Nmz0b59e5W6GzduxPLly3Ht2jVMmjQJp06dyjeYT57Bgwfj+vXr+OabbxAbG4u1a9fmuyx3zJgxOHHiBIYMGYJz587h+vXr2L59e6HzLIqzszNOnjyJuLg4JCcnF3nGsWrVqtiyZQvOnTuH8+fPo0ePHh/UGcrXpdNkMzk5Gbm5ubCzs1Mpt7OzQ0JCQrHtT506hUuXLqF///6F1gkLC4O5ubn0enmUKSIiIiIi0oypqSm8vb0xf/58NGnSBDVr1sTEiRMxYMAALFq0SKXulClTsG7dOtSuXRu//fYb/vjjD3h4eBQ430qVKmHz5s3Ytm0b6tSpg6VLl2LGjBkqdWrXro0jR47g2rVr8PHxwUcffYTQ0FCUL19e4+UYPXo09PX14eHhARsbmyLvv5w3bx4sLS3RqFEjBAQEwN/fH3Xr1tW4zw+NTAghdNX5P//8gwoVKuDEiRMqN/9+++23OHLkCE6ePFlk+0GDBiEqKqrAka/yZGVlqQzJnJaWBkdHR6SmpkKhULz5QhAVo+K28boOQavuBU7XdQhERETvhefPn+P27duoXLkyDA0NdR2O1slkMmzduhWBgYG6DoVeQ1HbZ1paGszNzYvNqXQ6QJC1tTX09fWRmJioUp6YmAh7e/si22ZkZGDdunWYOnVqkfXkcjnkcvkbx0pERERERETq0+lltAYGBvDy8kJERIRUplQqERERkW+Y41dt3LgRWVlZ6NWrV0mHSURERERERBrS+aNPQkJC0Lt3b9SrVw8NGjRAeHg4MjIyEBwcDAAICgpChQoVEBYWptLu119/RWBgIMqVK6eLsImIiIiIqAg6vFuP3hE6Tza7du2Khw8fIjQ0FAkJCfD09MTevXulQYPi4+Ohp6d6AjY2NhbHjh3D/v37dREyERERERERFUPnySYADBkypNDhiiMjI/OVVa9enb+UEBERERERvcN0es8mERERERERvZ+YbBIREREREZHWMdkkIiIiIiIirWOySURERERERFrHZJOIiIiIiKiEODs7Izw8XNdhFKikY3snRqMlIiIiIqL/q7ht/Fvr617gdI3b9OnTBykpKdi2bVuB052dnTFixAiMGDEi37S4uDhUrlwZenp6iI+PR4UKFaRpDx48gKOjI3Jzc3H79m04OzsDALZu3YpZs2YhJiYGSqUSlSpVQosWLVQSpWfPnmHmzJn4448/cOfOHZiZmaFZs2aYPHkyatSoIdWbPHkytm3bhnPnzmm83O+Cotbtu4ZnNomIiIiI6K2rUKECfvvtN5WyVatWqSSfABAREYGuXbuiY8eOOHXqFM6cOYPp06fjxYsXUp2srCz4+flh+fLl+O6773Dt2jXs3r0bOTk58Pb2xl9//fVWlolUMdkkIiIiIqK3rnfv3lixYoVK2YoVK9C7d2+Vsv/+979o3LgxvvnmG1SvXh3VqlVDYGAgFi9eLNUJDw9HVFQUdu7ciS5dusDJyQkNGjTA5s2b4e7ujn79+kEI8Vpxnj9/Hs2aNYOZmRkUCgW8vLzw999/S9OPHTsGHx8fGBkZwdHREcOGDUNGRkah80tJSUH//v1hY2MDhUKBTz/9FOfPn8+3zPXr14ehoSGsra3RoUMHAEDTpk1x584djBw5EjKZDDKZTO04kpKSEBAQACMjI1SuXBlr1qx5rfWhCSabRERERET01rVr1w5PnjzBsWPHAPybLD158gQBAQEq9ezt7XH58mVcunSp0HmtXbsWLVq0QJ06dVTK9fT0MHLkSFy5ciVfQqeunj17omLFijh9+jTOnDmDsWPHomzZsgCAmzdvomXLlujYsSMuXLiA9evX49ixYxgyZEih8+vcuTOSkpKwZ88enDlzBnXr1kXz5s3x+PFjAMCuXbvQoUMHtG7dGmfPnkVERAQaNGgAANiyZQsqVqyIqVOn4sGDB3jw4IHacfTp0wd3797F4cOHsWnTJvz4449ISkp6rXWiLt6zSUREREREb13ZsmXRq1cvLF++HJ988gmWL1+OXr16SYlcnqFDh+Lo0aOoVasWnJyc8PHHH+Ozzz5Dz549IZfLAQDXrl1Ds2bNCuzH3d1dquPp6alxnPHx8fjmm2/g5uYGAKhatao0LSwsDD179pTun6xatSp++OEH+Pr6YsmSJTA0NFSZ17Fjx3Dq1CkkJSVJsc+dOxfbtm3Dpk2bMHDgQEyfPh3dunXDlClTpHZ5SbSVlRX09fVhZmYGe3t7teOIj4/Hnj17cOrUKdSvXx8A8Ouvv0rrpqTwzCYREREREelE3759sXHjRiQkJGDjxo3o27dvvjomJibYtWsXbty4gQkTJsDU1BSjRo1CgwYNkJmZKdUr7jJZAwOD14oxJCQE/fv3h5+fH2bOnImbN29K086fP4+VK1fC1NRUevn7+0OpVOL27dv55nX+/Hmkp6ejXLlyKm1u374tzffcuXNo3ry5RjEWF0dMTAzKlCkDLy8vqY2bmxssLCxea52oi2c2iYiIiIhIJ2rVqgU3Nzd0794d7u7uqFmzZqGjxLq4uMDFxQX9+/fH+PHjUa1aNaxfvx7BwcGoWrUqYmJiCmyXV16tWrXXinHy5Mno0aMHdu3ahT179mDSpElYt24dOnTogPT0dAwaNAjDhg3L165SpUr5ytLT0+Hg4IDIyMh80/ISPyMjI41jLC6Oa9euaTxPbWCySUREREREOtO3b1989dVXWLJkidptnJ2dYWxsLA2A0717d4wfPx7nz59XuW9TqVRi/vz5qFevHjw8PF47xmrVqqFatWoYOXIkunfvjhUrVqBDhw6oW7curly5AldXV7XmU7duXSQkJKBMmTLSY11eVbt2bURERCA4OLjA6QYGBsjNzc0336LicHNzQ05ODs6cOSNdRhsbG4uUlBS14n5dTDaJiIiIiEhjqamp+c5ClitXDo6OjgCA+/fv55vu5OSUbz4DBgxA586dC72kc/LkycjMzETr1q3h5OSElJQU/PDDD3jx4gVatGgBABg5ciS2b9+OgIAAfP/99/D29kZiYiJmzJiB69ev48SJEyrzfPbsWb7YzMzM4OLikq/eN998g06dOqFy5cq4d+8eTp8+jY4dOwIAxowZg48//hhDhgxB//79YWJigitXruDAgQNYtGhRvmXx8/NDw4YNERgYiNmzZ6NatWr4559/pEGB6tWrh0mTJqF58+ZwcXFBt27dkJOTg927d2PMmDEA/k20//zzT3Tr1g1yuRzW1tbFxlG9enW0bNkSgwYNwpIlS1CmTBmMGDHitc6iaoL3bBIRERERkcYiIyPx0UcfqbxeHtRm7ty5+abv2rUr33zKlCkDa2trlClT8HkwX19f3Lp1C0FBQXBzc0OrVq2QkJCA/fv3o3r16gAAQ0NDREREICgoCOPGjYOLiwsaNGiAS5cu4dKlS/nOal67di1fbIMGDcrXt76+Ph49eoSgoCBUq1YNXbp0QatWraTlrF27No4cOYJr167Bx8cHH330EUJDQ1G+fPkCl0Umk2H37t1o0qQJgoODUa1aNXTr1g137tyBnZ0dgH8fb7Jx40bs2LEDnp6e+PTTT3Hq1ClpHlOnTkVcXBxcXFxgY2OjdhwrVqxA+fLl4evri88//xwDBw6Era1tgXFqi0y87gNnSqm0tDSYm5sjNTUVCoVC1+HQB6DitvG6DkGr7gVO13UIRERE74Xnz5/j9u3bqFy5cr5RS+nN7dmzBx06dMDcuXOLfBQJFayo7VPdnIpnNomIiIiI6L3TqlUr7NmzB48fP0ZycrKuw/kg8Z5NIiIiIiJ6LzVr1qzQ529SyeOZTSIiIiIiItI6JptERERERESkdUw2iYiIiIh06AMbr5NKCW1sl0w2iYiIiIh0oGzZsgCAzMxMHUdClF/edpm3nb4ODhBERERERKQD+vr6sLCwQFJSEgDA2NgYMplMx1HRh04IgczMTCQlJcHCwgL6+vqvPS8mm0REREREOmJvbw8AUsJJ9K6wsLCQts/XxWSTiIiIiEhHZDIZHBwcYGtrixcvXug6HCIA/146+yZnNPMw2SQiIiIi0jF9fX2t/HNP9C7R+QBBixcvhrOzMwwNDeHt7Y1Tp04VWT8lJQVff/01HBwcIJfLUa1aNezevfstRUtERERERETq0OmZzfXr1yMkJARLly6Ft7c3wsPD4e/vj9jYWNja2uarn52djRYtWsDW1habNm1ChQoVcOfOHVhYWLz94ImIiIiIiKhQOk02582bhwEDBiA4OBgAsHTpUuzatQvLly/H2LFj89Vfvnw5Hj9+jBMnTkhD8Do7O7/NkImIiIiIiEgNOruMNjs7G2fOnIGfn9//g9HTg5+fH6Kiogpss2PHDjRs2BBff/017OzsULNmTcyYMQO5ublvK2wiIiIiIiJSg87ObCYnJyM3Nxd2dnYq5XZ2drh69WqBbW7duoVDhw6hZ8+e2L17N27cuIGvvvoKL168wKRJkwpsk5WVhaysLOl9Wlqa9haCiIiIiIiICqTzAYI0oVQqYWtri59//hleXl7o2rUrxo8fj6VLlxbaJiwsDObm5tLL0dHxLUZMRERERET0YdJZsmltbQ19fX0kJiaqlCcmJhb68FAHBwdUq1ZNZVhod3d3JCQkIDs7u8A248aNQ2pqqvS6e/eu9haCiIiIiIiICqSzZNPAwABeXl6IiIiQypRKJSIiItCwYcMC2zRu3Bg3btyAUqmUyq5duwYHBwcYGBgU2EYul0OhUKi8iIiIiIiIqGTp9DLakJAQLFu2DKtWrUJMTAy+/PJLZGRkSKPTBgUFYdy4cVL9L7/8Eo8fP8bw4cNx7do17Nq1CzNmzMDXX3+tq0UgIiIiIiKiAuj00Sddu3bFw4cPERoaioSEBHh6emLv3r3SoEHx8fHQ0/t/Puzo6Ih9+/Zh5MiRqF27NipUqIDhw4djzJgxuloEIiIiIiIiKoBMCCF0HcTblJaWBnNzc6SmpvKSWnorKm4br+sQtOpe4HRdh0BEREREOqRuTlWqRqMlIiIiIiKi0oHJJhEREREREWkdk00iIiIiIiLSOiabREREREREpHVMNomIiIiIiEjrmGwSERERERGR1jHZJCIiIiIiIq1jsklERERERERax2STiIiIiIiItI7JJhEREREREWkdk00iIiIiIiLSOiabREREREREpHVMNomIiIiIiEjrmGwSERERERGR1jHZJCIiIiIiIq1jsklERERERERax2STiIiIiIiItI7JJhEREREREWkdk00iIiIiIiLSOiabREREREREpHVMNomIiIiIiEjrNE427969i3v37knvT506hREjRuDnn3/WamBERERERERUemmcbPbo0QOHDx8GACQkJKBFixY4deoUxo8fj6lTp2o9QCIiIiIiIip9NE42L126hAYNGgAANmzYgJo1a+LEiRNYs2YNVq5cqe34iIiIiIiIqBTSONl88eIF5HI5AODgwYNo164dAMDNzQ0PHjzQbnRERERERERUKmmcbNaoUQNLly7F0aNHceDAAbRs2RIA8M8//6BcuXJaD5CIiIiIiIhKH42TzVmzZuGnn35C06ZN0b17d9SpUwcAsGPHDunyWiIiIiIiIvqwaZxsNm3aFMnJyUhOTsby5cul8oEDB2Lp0qWvFcTixYvh7OwMQ0NDeHt749SpU4XWXblyJWQymcrL0NDwtfolIiIiIiKikqFxsrl8+XLEx8fD0tJSpdzZ2Rm2trYaB7B+/XqEhIRg0qRJiI6ORp06deDv74+kpKRC2ygUCjx48EB63blzR+N+iYiIiIiIqORonGyGhYXB1dUVlSpVwhdffIFffvkFN27ceO0A5s2bhwEDBiA4OBgeHh5YunQpjI2NVc6avkomk8He3l562dnZvXb/REREREREpH0aJ5vXr19HfHw8wsLCYGxsjLlz56J69eqoWLEievXqpdG8srOzcebMGfj5+f0/ID09+Pn5ISoqqtB26enpcHJygqOjI9q3b4/Lly8XWjcrKwtpaWkqLyIiIiIiIipZGiebAFChQgX07NkT8+fPx4IFC/DFF18gMTER69at02g+ycnJyM3NzXdm0s7ODgkJCQW2qV69OpYvX47t27fj999/h1KpRKNGjXDv3r0C64eFhcHc3Fx6OTo6ahQjERERERERaU7jZHP//v34z3/+g0aNGqFcuXIYN24cLC0tsWnTJjx8+LAkYlTRsGFDBAUFwdPTE76+vtiyZQtsbGzw008/FVh/3LhxSE1NlV53794t8RiJiIiIiIg+dGU0bdCyZUvY2Nhg1KhR2L17NywsLF67c2tra+jr6yMxMVGlPDExEfb29mrNo2zZsvjoo48KvW9ULpdDLpe/doxERERERESkOY3PbM6bNw+NGzfG7NmzUaNGDfTo0QM///wzrl27pnHnBgYG8PLyQkREhFSmVCoRERGBhg0bqjWP3NxcXLx4EQ4ODhr3T0RERERERCVD42RzxIgR2LJlC5KTk7F37140atQIe/fuRc2aNVGxYkWNAwgJCcGyZcuwatUqxMTE4Msvv0RGRgaCg4MBAEFBQRg3bpxUf+rUqdi/fz9u3bqF6Oho9OrVC3fu3EH//v017puIiIiIiIhKhsaX0QKAEAJnz55FZGQkDh8+jGPHjkGpVMLGxkbjeXXt2hUPHz5EaGgoEhIS4Onpib1790qDBsXHx0NP7/858ZMnTzBgwAAkJCTA0tISXl5eOHHiBDw8PF5nUYiIiIiIiKgEyIQQQpMGAQEBOH78ONLS0lCnTh00bdoUvr6+aNKkyRvdv/m2pKWlwdzcHKmpqVAoFLoOhz4AFbeN13UIWnUvcLquQyAiIiIiHVI3p9L4zKabmxsGDRoEHx8fmJubv1GQRERERERE9H7SONmcM2dOScRBRERERERE7xGNBwgCgCNHjiAgIACurq5wdXVFu3btcPToUW3HRkRERERERKWUxsnm77//Dj8/PxgbG2PYsGEYNmwYjIyM0Lx5c6xdu7YkYiQiIiIiIqJSRuMBgtzd3TFw4ECMHDlSpXzevHlYtmwZYmJitBqgtnGAIHrbOEAQEREREb1P1M2pND6zeevWLQQEBOQrb9euHW7fvq3p7IiIiIiIiOg9pHGy6ejoiIiIiHzlBw8ehKOjo1aCIiIiIiIiotJN49FoR40ahWHDhuHcuXNo1KgRAOD48eNYuXIlFixYoPUAiYiIiIiIqPTRONn88ssvYW9vj++//x4bNmwA8O99nOvXr0f79u21HiARERERERGVPholmzk5OZgxYwb69u2LY8eOlVRMREREREREVMppdM9mmTJlMHv2bOTk5JRUPERERERERPQe0HiAoObNm+PIkSMlEQsRERERERG9JzS+Z7NVq1YYO3YsLl68CC8vL5iYmKhMb9eundaCIyIiIiIiotJJ42Tzq6++AgDMmzcv3zSZTIbc3Nw3j4qIiIiIiIhKNY2TTaVSWRJxEBERERER0XtE43s2iYiIiIiIiIrDZJOIiIiIiIi0jskmERERERERaR2TTSIiIiIiItI6tZLNkJAQZGRkAAD+/PNP5OTklGhQREREREREVLqplWwuXLgQ6enpAIBmzZrh8ePHJRoUERERERERlW5qPfrE2dkZP/zwAz777DMIIRAVFQVLS8sC6zZp0kSrARIREREREVHpo1ayOWfOHAwePBhhYWGQyWTo0KFDgfVkMhlyc3O1GiARERERERGVPmolm4GBgQgMDER6ejoUCgViY2Nha2tb0rERERERERFRKaVWspnH1NQUhw8fRuXKlVGmjEZNiYiIiIiI6AOiccbo6+uL3NxcbN68GTExMQAADw8PtG/fHvr6+loPkIiIiIiIiEofjZPNGzduoE2bNrh37x6qV68OAAgLC4OjoyN27doFFxcXrQdJREREREREpYtajz552bBhw1ClShXcvXsX0dHRiI6ORnx8PCpXroxhw4a9VhCLFy+Gs7MzDA0N4e3tjVOnTqnVbt26dZDJZAgMDHytfomIiIiIiKhkaJxsHjlyBLNnz4aVlZVUVq5cOcycORNHjhzROID169cjJCQEkyZNQnR0NOrUqQN/f38kJSUV2S4uLg6jR4+Gj4+Pxn0SERERERFRydI42ZTL5Xj69Gm+8vT0dBgYGGgcwLx58zBgwAAEBwfDw8MDS5cuhbGxMZYvX15om9zcXPTs2RNTpkxBlSpVNO6TiIiIiIiISpbGyWbbtm0xcOBAnDx5EkIICCHw119/YfDgwWjXrp1G88rOzsaZM2fg5+f3/4D09ODn54eoqKhC202dOhW2trbo169fsX1kZWUhLS1N5UVEREREREQlS+Nk84cffoCLiwsaNmwIQ0NDGBoaonHjxnB1dcWCBQs0mldycjJyc3NhZ2enUm5nZ4eEhIQC2xw7dgy//vorli1bplYfYWFhMDc3l16Ojo4axUhERERERESa03g0WgsLC2zfvh03btyQHn3i7u4OV1dXrQf3qqdPn+KLL77AsmXLYG1trVabcePGISQkRHqflpbGhJOIiIiIiKiEaZxs5nF1dX3jBNPa2hr6+vpITExUKU9MTIS9vX2++jdv3kRcXBwCAgKkMqVSCQAoU6YMYmNj8z16RS6XQy6Xv1GcREREREREpBmNL6PVJgMDA3h5eSEiIkIqUyqViIiIQMOGDfPVd3Nzw8WLF3Hu3Dnp1a5dOzRr1gznzp3jGUsiIiIiIqJ3xGuf2dSWkJAQ9O7dG/Xq1UODBg0QHh6OjIwMBAcHAwCCgoJQoUIFhIWFwdDQEDVr1lRpb2FhAQD5yomIiIiIiEh3dJ5sdu3aFQ8fPkRoaCgSEhLg6emJvXv3SoMGxcfHQ09PpydgiYiIiIiISEMyIYTQdRBvU1paGszNzZGamgqFQqHrcOgDUHHbeF2HoFX3AqfrOgQiIiIi0iF1c6rXOmV49OhR9OrVCw0bNsT9+/cBAKtXr8axY8deL1oiIiIiIiJ6r2icbG7evBn+/v4wMjLC2bNnkZWVBQBITU3FjBkztB4gERERERERlT4aJ5vfffcdli5dimXLlqFs2bJSeePGjREdHa3V4IiIiIiIiKh00jjZjI2NRZMmTfKVm5ubIyUlRRsxERERERERUSmncbJpb2+PGzdu5Cs/duwYqlSpopWgiIiIiIiIqHTTONkcMGAAhg8fjpMnT0Imk+Gff/7BmjVrMHr0aHz55ZclESMRERERERGVMho/Z3Ps2LFQKpVo3rw5MjMz0aRJE8jlcowePRpDhw4tiRiJiIiIiIiolNEo2czNzcXx48fx9ddf45tvvsGNGzeQnp4ODw8PmJqallSMREREREREVMpolGzq6+vjs88+Q0xMDCwsLODh4VFScREREREREVEppvE9mzVr1sStW7dKIhYiIiIiIiJ6T7zWczZHjx6NnTt34sGDB0hLS1N5EREREREREWk8QFDr1q0BAO3atYNMJpPKhRCQyWTIzc3VXnRERERERERUKmmcbB4+fLgk4iAiIiIiIqL3iMbJpq+vb0nEQURERERERO8RjZPNPJmZmYiPj0d2drZKee3atd84KCIiIiIiIirdNE42Hz58iODgYOzZs6fA6bxnk4iIiIiIiDQejXbEiBFISUnByZMnYWRkhL1792LVqlWoWrUqduzYURIxEhERERERUSmj8ZnNQ4cOYfv27ahXrx709PTg5OSEFi1aQKFQICwsDG3atCmJOImIiIiIiKgU0fjMZkZGBmxtbQEAlpaWePjwIQCgVq1aiI6O1m50REREREREVCppnGxWr14dsbGxAIA6dergp59+wv3797F06VI4ODhoPUAiIiIiIiIqfTS+jHb48OF48OABAGDSpElo2bIl1qxZAwMDA6xcuVLb8REREREREVEppHGy2atXL+lvLy8v3LlzB1evXkWlSpVgbW2t1eCIiIiIiIiodHrt52zmMTY2Rt26dbURCxEREREREb0nNE42c3NzsXLlSkRERCApKQlKpVJl+qFDh7QWHBEREREREZVOr3XP5sqVK9GmTRvUrFkTMpmsJOIiIiIiIiKiUkzjZHPdunXYsGEDWrduXRLxEBERERER0XtA40efGBgYwNXVtSRiISIiIiIioveExsnmqFGjsGDBAgghtBbE4sWL4ezsDENDQ3h7e+PUqVOF1t2yZQvq1asHCwsLmJiYwNPTE6tXr9ZaLERERERERPTm1LqM9vPPP1d5f+jQIezZswc1atRA2bJlVaZt2bJFowDWr1+PkJAQLF26FN7e3ggPD4e/vz9iY2Nha2ubr76VlRXGjx8PNzc3GBgYYOfOnQgODoatrS38/f016puIiIiIiIhKhkyocYoyODhY7RmuWLFCowC8vb1Rv359LFq0CACgVCrh6OiIoUOHYuzYsWrNo27dumjTpg2mTZtWbN20tDSYm5sjNTUVCoVCo1iJXkfFbeN1HYJW3QucrusQiIiIiEiH1M2p1DqzqWkCqa7s7GycOXMG48aNk8r09PTg5+eHqKioYtsLIXDo0CHExsZi1qxZBdbJyspCVlaW9D4tLe3NAyciIiIiIqIiaXzP5rNnz5CZmSm9v3PnDsLDw7F//36NO09OTkZubi7s7OxUyu3s7JCQkFBou9TUVJiamsLAwABt2rTBwoUL0aJFiwLrhoWFwdzcXHo5OjpqHCcRERERERFpRuNks3379vjtt98AACkpKWjQoAG+//57tG/fHkuWLNF6gAUxMzPDuXPncPr0aUyfPh0hISGIjIwssO64ceOQmpoqve7evftWYiQiIiIiIvqQaZxsRkdHw8fHBwCwadMm2Nvb486dO/jtt9/www8/aDQva2tr6OvrIzExUaU8MTER9vb2hQetpwdXV1d4enpi1KhR6NSpE8LCwgqsK5fLoVAoVF5ERERERERUsjRONjMzM2FmZgYA2L9/Pz7//HPo6enh448/xp07dzSal4GBAby8vBARESGVKZVKREREoGHDhmrPR6lUqtyXSURERERERLqlcbLp6uqKbdu24e7du9i3bx8+++wzAEBSUtJrnTUMCQnBsmXLsGrVKsTExODLL79ERkaGNAJuUFCQygBCYWFhOHDgAG7duoWYmBh8//33WL16NXr16qVx30RERERERFQy1BqN9mWhoaHo0aMHRo4ciebNm0tnIPfv34+PPvpI4wC6du2Khw8fIjQ0FAkJCfD09MTevXulQYPi4+Ohp/f/nDgjIwNfffUV7t27ByMjI7i5ueH3339H165dNe6biIiIiIiISoZaz9l8VUJCAh48eIA6depIieCpU6egUCjg5uam9SC1ic/ZpLeNz9kkIiIioveJVp+z+Sp7e/t8A/g0aNDgdWZFRERERERE7yGN79kkIiIiIiIiKg6TTSIiIiIiItI6JptERERERESkdUw2iYiIiIiISOuYbBIREREREZHWMdkkIiIiIiIirWOySURERERERFrHZJOIiIiIiIi0jskmERERERERaR2TTSIiIiIiItI6JptERERERESkdUw2iYiIiIiISOuYbBIREREREZHWMdkkIiIiIiIirWOySURERERERFrHZJOIiIiIiIi0jskmERERERERaR2TTSIiIiIiItI6JptERERERESkdUw2iYiIiIiISOuYbBIREREREZHWMdkkIiIiIiIirWOySURERERERFrHZJOIiIiIiIi0jskmERERERERad07kWwuXrwYzs7OMDQ0hLe3N06dOlVo3WXLlsHHxweWlpawtLSEn59fkfWJiIiIiIjo7dN5srl+/XqEhIRg0qRJiI6ORp06deDv74+kpKQC60dGRqJ79+44fPgwoqKi4OjoiM8++wz3799/y5ETERERERFRYWRCCKHLALy9vVG/fn0sWrQIAKBUKuHo6IihQ4di7NixxbbPzc2FpaUlFi1ahKCgoGLrp6WlwdzcHKmpqVAoFG8cP1FxKm4br+sQtOpe4HRdh0BE9F56374vAH5nEL2v1M2pdHpmMzs7G2fOnIGfn59UpqenBz8/P0RFRak1j8zMTLx48QJWVlYlFSYRERERERFpqIwuO09OTkZubi7s7OxUyu3s7HD16lW15jFmzBiUL19eJWF9WVZWFrKysqT3aWlprx8wERERERERqUXn92y+iZkzZ2LdunXYunUrDA0NC6wTFhYGc3Nz6eXo6PiWoyQiIiIiIvrw6DTZtLa2hr6+PhITE1XKExMTYW9vX2TbuXPnYubMmdi/fz9q165daL1x48YhNTVVet29e1crsRMREREREVHhdJpsGhgYwMvLCxEREVKZUqlEREQEGjZsWGi72bNnY9q0adi7dy/q1atXZB9yuRwKhULlRURERERERCVLp/dsAkBISAh69+6NevXqoUGDBggPD0dGRgaCg4MBAEFBQahQoQLCwsIAALNmzUJoaCjWrl0LZ2dnJCQkAABMTU1hamqqs+UgIiIiIiKi/9N5stm1a1c8fPgQoaGhSEhIgKenJ/bu3SsNGhQfHw89vf+fgF2yZAmys7PRqVMnlflMmjQJkydPfpuhExERERERUSF0nmwCwJAhQzBkyJACp0VGRqq8j4uLK/mAiIiIiIiI6I2U6tFoiYiIiIiI6N3EZJOIiIiIiIi0jskmERERERERaR2TTSIiIiIiItI6JptERERERESkdUw2iYiIiIiISOuYbBIREREREZHWMdkkIiIiIiIirWOySURERERERFrHZJOIiIiIiIi0jskmERERERERaR2TTSIiIiIiItI6JptERERERESkdUw2iYiIiIiISOuYbBIREREREZHWMdkkIiIiIiIirWOySURERERERFrHZJOIiIiIiIi0jskmERERERERaR2TTSIiIiIiItI6JptERERERESkdUw2iYiIiIiISOuYbBIREREREZHWMdkkIiIiIiIirWOySURERERERFrHZJOIiIiIiIi0jskmERERERERaZ3Ok83FixfD2dkZhoaG8Pb2xqlTpwqte/nyZXTs2BHOzs6QyWQIDw9/e4ESERERERGR2nSabK5fvx4hISGYNGkSoqOjUadOHfj7+yMpKanA+pmZmahSpQpmzpwJe3v7txwtERERERERqUunyea8efMwYMAABAcHw8PDA0uXLoWxsTGWL19eYP369etjzpw56NatG+Ry+VuOloiIiIiIiNSls2QzOzsbZ86cgZ+f3/+D0dODn58foqKitNZPVlYW0tLSVF5ERERERERUsnSWbCYnJyM3Nxd2dnYq5XZ2dkhISNBaP2FhYTA3N5dejo6OWps3ERERERERFUznAwSVtHHjxiE1NVV63b17V9chERERERERvffK6Kpja2tr6OvrIzExUaU8MTFRq4P/yOVy3t9JRERERET0luks2TQwMICXlxciIiIQGBgIAFAqlYiIiMCQIUN0FRYREREREb0jKm4br+sQtO5e4HRdh/DW6CzZBICQkBD07t0b9erVQ4MGDRAeHo6MjAwEBwcDAIKCglChQgWEhYUB+HdQoStXrkh/379/H+fOnYOpqSlcXV11thxERERERESkSqfJZteuXfHw4UOEhoYiISEBnp6e2Lt3rzRoUHx8PPT0/n9b6T///IOPPvpIej937lzMnTsXvr6+iIyMfNvhExERERERUSF0mmwCwJAhQwq9bPbVBNLZ2RlCiLcQFREREREREb2J9340WiIiIiIiInr7mGwSERERERGR1jHZJCIiIiIiIq1jsklERERERERap/MBgoiI6MPD56YRERG9/3hmk4iIiIiIiLSOySYRERERERFpHZNNIiIiIiIi0jomm0RERERERKR1TDaJiIiIiIhI65hsEhERERERkdYx2SQiIiIiIiKtY7JJREREREREWldG1wHQ66sxLkbXIWjd5TB3XYdARERERERawDObREREREREpHVMNomIiIiIiEjrmGwSERERERGR1jHZJCIiIiIiIq1jsklERERERERax2STiIiIiIiItI7JJhEREREREWkdk00iIiIiIiLSOiabREREREREpHVMNomIiIiIiEjryug6ACIibaoxLkbXIWjd5TB3XYdAREREpDGe2SQiIiIiIiKtY7JJREREREREWvdOJJuLFy+Gs7MzDA0N4e3tjVOnThVZf+PGjXBzc4OhoSFq1aqF3bt3v6VIiYiIiIiISB06v2dz/fr1CAkJwdKlS+Ht7Y3w8HD4+/sjNjYWtra2+eqfOHEC3bt3R1hYGNq2bYu1a9ciMDAQ0dHRqFmzpg6WgIiIiIhKo/ftPn/e40/vGp2f2Zw3bx4GDBiA4OBgeHh4YOnSpTA2Nsby5csLrL9gwQK0bNkS33zzDdzd3TFt2jTUrVsXixYtesuRExERERERUWF0emYzOzsbZ86cwbhx46QyPT09+Pn5ISoqqsA2UVFRCAkJUSnz9/fHtm3bCqyflZWFrKws6X1qaioAIC0t7Q2j173crHRdh6B178Pn8iplZlbxlUqRd/0z4n5ROrxv+wXwfn5O9HZxv3j73rfvjHd9fb8O7hfvprxlEEIUWU+nyWZycjJyc3NhZ2enUm5nZ4erV68W2CYhIaHA+gkJCQXWDwsLw5QpU/KVOzo6vmbUVJLM5+s6AiqOOb7XdQgfHO4XpQP3DaL8uF+8Xfy+KB3ep/3i6dOnMDc3L3S6zu/ZLGnjxo1TOROqVCrx+PFjlCtXDjKZTIeRka6kpaXB0dERd+/ehUKh0HU4RO8E7hdEBeO+QZQf9wsSQuDp06coX758kfV0mmxaW1tDX18fiYmJKuWJiYmwt7cvsI29vb1G9eVyOeRyuUqZhYXF6wdN7w2FQsEDJNEruF8QFYz7BlF+3C8+bEWd0cyj0wGCDAwM4OXlhYiICKlMqVQiIiICDRs2LLBNw4YNVeoDwIEDBwqtT0RERERERG+fzi+jDQkJQe/evVGvXj00aNAA4eHhyMjIQHBwMAAgKCgIFSpUQFhYGABg+PDh8PX1xffff482bdpg3bp1+Pvvv/Hzzz/rcjGIiIiIiIjoJTpPNrt27YqHDx8iNDQUCQkJ8PT0xN69e6VBgOLj46Gn9/8TsI0aNcLatWsxYcIE/Oc//0HVqlWxbds2PmOT1CaXyzFp0qR8l1cTfci4XxAVjPsGUX7cL0hdMlHceLVEREREREREGtLpPZtERERERET0fmKySURERERERFrHZJOIiIiIiIi0jskmEdE7qk+fPggMDFS7flxcHGQyGc6dO/fW+ybSRNOmTTFixIi32oezszPCw8NLtM/irFy5UuVZ35MnT4anp6dG83gb647oTZSG749X90UqOTofjZaIiAq2YMECcAy3/CIjI9GsWTM8efKE/yyQ2k6fPg0TExNdh6Fi9OjRGDp0qEZttmzZgrJly0rvnZ2dMWLECCagRPROYrJJRPSOMjc313UIWiWEQG5uLsqUeTe+et61eKhk2djY6DqEfExNTWFqaqpRGysrqxKKhnQlOzsbBgYGb7XPFy9eqPxo8b7SxbolVbyMlkqNTZs2oVatWjAyMkK5cuXg5+eHjIwM6XKNKVOmwMbGBgqFAoMHD0Z2drbUdu/evfjkk09gYWGBcuXKoW3btrh586bK/O/du4fu3bvDysoKJiYmqFevHk6ePClN3759O+rWrQtDQ0NUqVIFU6ZMQU5OzltbftKNN9nulEolwsLCULlyZRgZGaFOnTrYtGmTyvwvX76Mtm3bQqFQwMzMDD4+PtK2+eqlSOpsx5ooqu88c+fOhYODA8qVK4evv/4aL168kKatXr0a9erVg5mZGezt7dGjRw8kJSVJ0yMjIyGTybBnzx54eXlBLpfj2LFjuHnzJtq3bw87OzuYmpqifv36OHjwoEq/WVlZGDNmDBwdHSGXy+Hq6opff/0VcXFxaNasGQDA0tISMpkMffr0UWt9FxYP6VZWVhZGjx6NChUqwMTEBN7e3oiMjJSmP3r0CN27d0eFChVgbGyMWrVq4Y8//lCZR0ZGBoKCgmBqagoHBwd8//33+fp59TJamUyGX375BR06dICxsTGqVq2KHTt2qLTZsWMHqlatCkNDQzRr1gyrVq2CTCZDSkqKWsu2cuVKVKpUCcbGxujQoQMePXqkMv3Vy2hzcnIwbNgwaR8fM2YMevfurXIcePky2qZNm+LOnTsYOXIkZDIZZDKZWnGRbjVt2hRDhgzBiBEjYG1tDX9/f1y6dAmtWrWCqakp7Ozs8MUXXyA5OVlqU9h3UZ5ffvkF7u7uMDQ0hJubG3788UdpWt4tFuvXr4evry8MDQ2xZMkSGBkZYc+ePSqxbd26FWZmZsjMzAQA3L17F126dIGFhQWsrKzQvn17xMXFSfVzc3MREhIibbPffvut2lfk7Ny5ExYWFsjNzQUAnDt3DjKZDGPHjpXq9O/fH7169ZLeb968GTVq1IBcLoezs3O+fd3Z2RnTpk1DUFAQFAoFBg4cCKD4fZFKkCAqBf755x9RpkwZMW/ePHH79m1x4cIFsXjxYvH06VPRu3dvYWpqKrp27SouXbokdu7cKWxsbMR//vMfqf2mTZvE5s2bxfXr18XZs2dFQECAqFWrlsjNzRVCCPH06VNRpUoV4ePjI44ePSquX78u1q9fL06cOCGEEOLPP/8UCoVCrFy5Uty8eVPs379fODs7i8mTJ+tkfdDb8abb3XfffSfc3NzE3r17xc2bN8WKFSuEXC4XkZGRQggh7t27J6ysrMTnn38uTp8+LWJjY8Xy5cvF1atXhRBC9O7dW7Rv316aX3Hb8e3btwUAcfbs2WKXTZ2+FQqFGDx4sIiJiRH//e9/hbGxsfj555+lefz6669i9+7d4ubNmyIqKko0bNhQtGrVSpp++PBhAUDUrl1b7N+/X9y4cUM8evRInDt3TixdulRcvHhRXLt2TUyYMEEYGhqKO3fuSG27dOkiHB0dxZYtW8TNmzfFwYMHxbp160ROTo7YvHmzACBiY2PFgwcPREpKilrru7B46O3z9fUVw4cPF0II0b9/f9GoUSPx559/ihs3bog5c+YIuVwurl27JoT4d1udM2eOOHv2rLh586b44YcfhL6+vjh58qQ0vy+//FJUqlRJHDx4UFy4cEG0bdtWmJmZSX0IIYSTk5OYP3++9B6AqFixoli7dq24fv26GDZsmDA1NZW2iVu3bomyZcuK0aNHi6tXr4o//vhDVKhQQQAQT548KXYZ//rrL6GnpydmzZolYmNjxYIFC4SFhYUwNzeX6kyaNEnUqVNHev/dd98JKysrsWXLFhETEyMGDx4sFAqFynHg5XX36NEjUbFiRTF16lTx4MED8eDBA7XWP+mWr6+vMDU1Fd988424evWq+Ouvv4SNjY0YN26ciImJEdHR0aJFixaiWbNmQoiiv4uEEOL3338XDg4OYvPmzeLWrVti8+bNwsrKSqxcuVII8f/vBmdnZ6nOP//8Izp16iR69eqlElvHjh2lsuzsbOHu7i769u0rLly4IK5cuSJ69OghqlevLrKysoQQQsyaNUtYWlqKzZs3iytXroh+/foJMzMzlW22MCkpKUJPT0+cPn1aCCFEeHi4sLa2Ft7e3lIdV1dXsWzZMiGEEH///bfQ09MTU6dOFbGxsWLFihXCyMhIrFixQqrv5OQkFAqFmDt3rrhx44a4ceOGWvsilRwmm1QqnDlzRgAQcXFx+ab17t1bWFlZiYyMDKlsyZIlwtTUVPon/FUPHz4UAMTFixeFEEL89NNPwszMrNB/PJs3by5mzJihUrZ69Wrh4ODwuotEpcCbbHfPnz8XxsbG0g8Wefr16ye6d+8uhBBi3LhxonLlyiI7O7vA/l9NNl/16nasSbKpTt9OTk4iJydHKuvcubPo2rVrofM8ffq0ACD9A5SX3G3btq3YeGrUqCEWLlwohBAiNjZWABAHDhwosG7efF/+h1+d9a1JPFSy8hKmO3fuCH19fXH//n2V6c2bNxfjxo0rtH2bNm3EqFGjhBD//lhoYGAgNmzYIE1/9OiRMDIyKjbZnDBhgvQ+PT1dABB79uwRQggxZswYUbNmTZV+x48fr3ay2b17d9G6dWuVsq5duxaZbNrZ2Yk5c+ZI73NyckSlSpUKTTYLWi569/n6+oqPPvpIej9t2jTx2WefqdS5e/eu9KNaUd9FQgjh4uIi1q5dq1I2bdo00bBhQyHE/78bwsPDVeps3bpVmJqaSt9jqampwtDQUNoHVq9eLapXry6USqXUJisrSxgZGYl9+/YJIYRwcHAQs2fPlqa/ePFCVKxYUa1kUwgh6tatK23zgYGBYvr06cLAwEA8ffpU3Lt3TwCQfnjq0aOHaNGihUr7b775Rnh4eEjvnZycRGBgoEoddfZFKjm8jJZKhTp16qB58+aoVasWOnfujGXLluHJkycq042NjaX3DRs2RHp6Ou7evQsAuH79Orp3744qVapAoVDA2dkZABAfHw/g30s3Pvroo0LvhTl//jymTp0q3V9jamqKAQMG4MGDB9KlJvT+eZPt7saNG8jMzESLFi1UtpvffvtNulT13Llz8PHxUfu+meK2Y02o03eNGjWgr68vvXdwcFC5TPbMmTMICAhApUqVYGZmBl9f3wLjqVevnsr79PR0jB49Gu7u7rCwsICpqSliYmJU9kd9fX1pfupQZ30XFg/pzsWLF5Gbm4tq1aqpfG5HjhyRPrfc3FxMmzYNtWrVgpWVFUxNTbFv3z5pe7l58yays7Ph7e0tzdfKygrVq1cvtv/atWtLf5uYmEChUEjbeGxsLOrXr69Sv0GDBmovW0xMjEpMwL/HiMKkpqYiMTFRpQ99fX14eXmp3SeVHi9/rufPn8fhw4dV9gE3NzcA/27fRX0XZWRk4ObNm+jXr59K+++++67YY1/r1q1RtmxZ6fLxzZs3Q6FQwM/PT4rrxo0bMDMzk+ZrZWWF58+f4+bNm0hNTcWDBw9UtvMyZcpodIz19fVFZGQkhBA4evQoPv/8c7i7u+PYsWM4cuQIypcvj6pVqwL4d59q3LixSvvGjRvj+vXr0qW4BS2npvsiaRdHRaBSQV9fHwcOHMCJEyewf/9+LFy4EOPHj1e5p7IoAQEBcHJywrJly1C+fHkolUrUrFlTur/OyMioyPbp6emYMmUKPv/883zTDA0NNV8gKhXeZLtLT08HAOzatQsVKlRQmSaXywEUv929qrjtWBPq9P1qIiqTyaBUKgH8+w+Ov78//P39sWbNGtjY2CA+Ph7+/v754nl1BNDRo0fjwIEDmDt3LlxdXWFkZIROnTqpvT8WRJ31XVg8pDvp6enQ19fHmTNnVH7YACANnDNnzhwsWLAA4eHhqFWrFkxMTDBixIjX2u5fVdQ2TlSSXj4OpaenIyAgALNmzcpXz8HBocjvorwfPJctW5YvoXp1n3r12GdgYIBOnTph7dq16NatG9auXYuuXbtKg6alp6fDy8sLa9asyReXtgbcatq0KZYvX47z58+jbNmycHNzQ9OmTREZGYknT55o9KNjHh7j3y1MNqnUkMlkaNy4MRo3bozQ0FA4OTlh69atAP799e3Zs2fSP6l//fUXTE1N4ejoiEePHiE2NhbLli2Dj48PAOQbFKR27dr45Zdf8Pjx4wLPbtatWxexsbFwdXUt4aWkd83rbndWVlaQy+WIj48v9Muydu3aWLVqlVqjAqqzHWtCk74LcvXqVTx69AgzZ86Eo6MjAODvv/9Wq+3x48fRp08fdOjQAcC//9C8POBErVq1oFQqceTIEekX9pfljSz48i/ZHh4exa5vevd89NFHyM3NRVJSkrRdv+r48eNo3769NEiIUqnEtWvX4OHhAQBwcXFB2bJlcfLkSVSqVAkA8OTJE1y7du2NtoXq1atj9+7dKmWnT59Wu727u3u+H6b++uuvQuubm5vDzs4Op0+fRpMmTQD8u41HR0cX+SxOAwMDlX2BSp+6deti8+bNcHZ2LnR07MK+i0JCQlC+fHncunULPXv21Ljvnj17okWLFrh8+TIOHTqE7777TiWu9evXw9bWFgqFosD2Dg4OOHnypLTN5uTk4MyZM6hbt65a/fv4+ODp06eYP3++tL82bdoUM2fOxJMnTzBq1Ciprru7O44fP67S/vjx46hWrVq+xPplmu6LpF28jJZKhZMnT2LGjBn4+++/ER8fjy1btuDhw4dwd3cH8O/Q1v369cOVK1ewe/duTJo0CUOGDIGenh4sLS1Rrlw5/Pzzz7hx4wYOHTqEkJAQlfl3794d9vb2CAwMxPHjx3Hr1i1s3rwZUVFRAIDQ0FD89ttvmDJlCi5fvoyYmBisW7cOEyZMeOvrgt6eN9nuzMzMMHr0aIwcORKrVq3CzZs3ER0djYULF2LVqlUAgCFDhiAtLQ3dunXD33//jevXr2P16tWIjY3NF4s627EmNOm7IJUqVYKBgQEWLlyIW7duYceOHZg2bZpabatWrYotW7bg3LlzOH/+PHr06KFyNsnZ2Rm9e/dG3759sW3bNty+fRuRkZHYsGEDAMDJyQkymQw7d+7Ew4cPkZ6ertb6pndPtWrV0LNnTwQFBWHLli24ffs2Tp06hbCwMOzatQvAv9tL3lmdmJgYDBo0CImJidI8TE1N0a9fP3zzzTc4dOgQLl26hD59+kBP783+xRk0aBCuXr2KMWPG4Nq1a9iwYQNWrlwJAGqN+jps2DDs3bsXc+fOxfXr17Fo0SLs3bu3yDZDhw5FWFgYtm/fjtjYWAwfPhxPnjwpsj9nZ2f8+eefuH//vsropVR6fP3113j8+DG6d++O06dP4+bNm9i3bx+Cg4ORm5tb7HfRlClTEBYWhh9++AHXrl3DxYsXsWLFCsybN6/Yvps0aQJ7e3v07NkTlStXVjk72rNnT1hbW6N9+/Y4evSodCweNmwY7t27BwAYPnw4Zs6ciW3btuHq1av46quv1B6tGfj3u6127dpYs2YNmjZtKsUUHR2d7wejUaNGISIiAtOmTcO1a9ewatUqLFq0CKNHjy6yj9fZF0mLdH3TKJE6rly5Ivz9/YWNjY2Qy+WiWrVq0mAieYOohIaGinLlyglTU1MxYMAA8fz5c6n9gQMHhLu7u5DL5aJ27doiMjJSABBbt26V6sTFxYmOHTsKhUIhjI2NRb169VRGO9y7d69o1KiRMDIyEgqFQjRo0EBlZE56/7zpdqdUKkV4eLioXr26KFu2rLCxsRH+/v7iyJEjUp3z58+Lzz77TBgbGwszMzPh4+Mjbt68qdJHnuK2Y00GCNK0byGEGD58uPD19ZXer127Vjg7Owu5XC4aNmwoduzYodJ/QQP55MXZrFkzYWRkJBwdHcWiRYvyDXry7NkzMXLkSOHg4CAMDAyEq6urWL58uTR96tSpwt7eXshkMtG7d2+11ndh8dDb9/LnnZ2dLUJDQ4Wzs7MoW7ascHBwEB06dBAXLlwQQvw72E/79u2FqampsLW1FRMmTBBBQUEq2+fTp09Fr169hLGxsbCzsxOzZ88udiCdV78DhBDC3NxcZWTL7du3C1dXVyGXy0XTpk3FkiVLBADx7NkztZbz119/FRUrVhRGRkYiICBAzJ07t8gBgl68eCGGDBkiFAqFsLS0FGPGjBGdO3cW3bp1K3DdCSFEVFSUqF27tpDL5YL/1pUOr36GQghx7do10aFDB2FhYSGMjIyEm5ubGDFihFAqlUV+F+VZs2aN8PT0FAYGBsLS0lI0adJEbNmyRQhR/HfDt99+KwCI0NDQfNMePHgggoKChLW1tZDL5aJKlSpiwIABIjU1VQjx7zY7fPhwoVAohIWFhQgJCcm3fxZn+PDhAoCIiYmRyurUqSPs7e3z1d20aZPw8PAQZcuWFZUqVVIZUEuIwgfMKm5fpJIjE0LNh+EQvaP69OmDlJQUbNu2Tdeh0AeE2x3Rh2f69OlYunSpNPhcSVMqlXB3d0eXLl3UvnKAiOhdwns2iYiIiArw448/on79+ihXrhyOHz+OOXPmYMiQISXW3507d7B//374+voiKysLixYtwu3bt9GjR48S65OIqCTxnk0iovfQ4MGDVYbBf/k1ePBgXYdHVCpcv34d7du3h4eHB6ZNm4ZRo0Zh8uTJAIBWrVoVuo/NmDHjtfrT09PDypUrUb9+fTRu3BgXL17EwYMHpXvziEqT+Pj4QvcRU1PT13psF5U+vIyWiOg9lJSUhLS0tAKnKRQK2NravuWIiN4v9+/fx7NnzwqcZmVlVehzm4k+FDk5OSojjb+qqNF36f3BZJOIiIiIiIi0jpfREhERERERkdYx2SQiIiIiIiKtY7JJREREREREWsdkk4iIiIiIiLSOySYRERERERFpHZNNIiIiIiIi0jomm0RERERERKR1TDaJiIiIiIhI6/4H69AhzEZLcEcAAAAASUVORK5CYII=\n"},"metadata":{}},{"output_type":"display_data","data":{"text/plain":" BIRD train Spider train LLMSQL selected\nspace 0.047 0.0 0.777\nspecial_character 0.019 0.0 0.071\nleading_digit 0.000 0.0 0.206\nreserved_word 0.008 0.0 0.094","text/html":"
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
BIRD trainSpider trainLLMSQL selected
space0.0470.00.777
special_character0.0190.00.071
leading_digit0.0000.00.206
reserved_word0.0080.00.094
\n
"},"metadata":{}}],"execution_count":15},{"id":"cell-24","cell_type":"markdown","source":"## 9. Save\n\nThe rows use the same column names as the BIRD trainset (`db_id`, `question`, `evidence`, `sql`, `schema`), where `db_id` is the LLMSQL table id, plus `sql_original` and the metadata used in the selection. Each selected table is exported to its own SQLite file, in the same `db_id/db_id.sqlite` layout as the other datasets.","metadata":{}},{"id":"cell-25","cell_type":"code","source":"selected = selected.rename(columns={\"table_id\": \"db_id\"})\noutput_columns = [\n \"db_id\", \"question\", \"evidence\", \"sql\", \"sql_original\", \"schema\", \"total_tokens\",\n \"quoted_identifiers\", \"quoting_reasons\", \"selection_reason\", \"aggregation\", \"n_result_rows\",\n]\nselected[output_columns].to_parquet(OUTPUT_DIR / \"llmsql_selected.parquet\", index=False)\n\nexport_dir = DATA_DIR / \"export\"\nwith zipfile.ZipFile(OUTPUT_DIR / \"llmsql_selected_databases.zip\", \"w\", zipfile.ZIP_DEFLATED) as archive:\n for db_id in tqdm(selected[\"db_id\"], desc=\"Exporting tables\"):\n target = export_dir / db_id / f\"{db_id}.sqlite\"\n target.parent.mkdir(parents=True, exist_ok=True)\n target.unlink(missing_ok=True)\n connection = sqlite3.connect(target)\n connection.execute(\"ATTACH DATABASE ? AS source\", (str(db_path),))\n connection.execute(tables.at[db_id, \"ddl\"])\n connection.execute(f\"INSERT INTO main.{quote_name(db_id)} SELECT * FROM source.{quote_name(db_id)}\")\n connection.commit()\n connection.close()\n archive.write(target, f\"{db_id}/{db_id}.sqlite\")\n\nprint(f\"Saved {len(selected):,} rows and {selected['db_id'].nunique():,} databases to {OUTPUT_DIR}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-10-20T00:40:19.918728Z","iopub.execute_input":"2026-10-20T00:40:19.919001Z","iopub.status.idle":"2026-10-20T00:46:09.952089Z","shell.execute_reply.started":"2026-10-20T00:40:19.918977Z","shell.execute_reply":"2026-10-20T00:46:09.951126Z"}},"outputs":[{"output_type":"display_data","data":{"text/plain":"Exporting tables: 0%| | 0/1000 [00:00