Experiment-LML.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Copyright &copy; 2024 Praneeth Vadlapati"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<!-- ## Setup:\n",
    "\n",
    "Example of .env: \n",
    "```python\n",
    "# Use any OpenAI-compatible API provider\n",
    "# Groq is preferred for llama3.1 due to free fast responses\n",
    "LM_PROVIDER_BASE_URL=https://api.groq.com/openai/v1\n",
    "LM_API_KEY=\n",
    "LM_MODEL=llama-3.1-70b-versatile\n",
    "```\n",
    "\n",
    "Installing packages:\n",
    "```bash\n",
    "pip install openai python-dotenv ucimlrepo tiktoken\n",
    "``` -->"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A. \tLoading a Dataset and Setting Up a Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from io import StringIO\n",
    "\n",
    "from IPython.display import display, Markdown\n",
    "from ucimlrepo import fetch_ucirepo\n",
    "import pandas as pd\n",
    "import tiktoken\n",
    "from common_functions import get_lm_response, extract_data, \\\n",
    "    print_progress, print_error, model, data_folder, random_state\n",
    "\n",
    "dataset_name = 'Rice (Cammeo and Osmancik)'\n",
    "# 'Iris', 'Wine', 'Zoo', 'Raisin'\n",
    "# 'Rice (Cammeo and Osmancik)'\n",
    "# 'Statlog (German Credit Data)'\n",
    "# 'Mushroom'\n",
    "\n",
    "filename = os.path.join(data_folder, f'{dataset_name}.csv')\n",
    "labelcol_filename = os.path.join(data_folder, f'{dataset_name}_labelcol.txt')\n",
    "\n",
    "try:  # Try to load from the local files\n",
    "\tif not os.path.exists(filename):\n",
    "\t\traise FileNotFoundError()\n",
    "\tdata_df = pd.read_csv(filename)\n",
    "\twith open(labelcol_filename, 'r') as file:\n",
    "\t\tlabel_col = file.read().strip()\n",
    "\t\tif not label_col:\n",
    "\t\t\terr = f'Label column not found in {labelcol_filename}'\n",
    "\t\t\tprint(err)\n",
    "\t\t\traise ValueError(err)\n",
    "\t\tif label_col not in data_df.columns:\n",
    "\t\t\terr = f'Label column {label_col} not found in dataset'\n",
    "\t\t\tprint(err)\n",
    "\t\t\traise ValueError(err)\n",
    "except Exception as e:\n",
    "\tprint('Loading data from UCI repository')\n",
    "\tdataset = fetch_ucirepo(name=dataset_name)\n",
    "\t# Extract column names from metadata\n",
    "\tfeature_columns = dataset.variables[dataset.variables['role'].str.lower() == 'feature']['name'].tolist()\n",
    "\ttarget_columns = dataset.variables[dataset.variables['role'].str.lower() == 'target']['name'].tolist()\n",
    "\n",
    "\tdata_df = pd.concat([\n",
    "\t\tpd.DataFrame(dataset.data.features, columns=feature_columns),  # df_features,\n",
    "\t\tpd.DataFrame(dataset.data.targets, columns=target_columns),  # df_targets,\n",
    "\t], axis=1)\n",
    "\t# In column names, replace spaces with underscores\n",
    "\tdata_df.columns = data_df.columns.str.replace(' ', '_').str.replace('-', '_')\n",
    "\tdata_df.to_csv(filename, index=False)\n",
    "\n",
    "\tlabel_col = target_columns[-1].replace(' ', '_').replace('-', '_')\n",
    "\twith open(labelcol_filename, 'w') as file:\n",
    "\t\tfile.write(label_col)\n",
    "\n",
    "predicted_col = f'{label_col}_predicted'\n",
    "train_data = f'```csv\\n{data_df.to_csv(index=False).strip()}\\n```'\n",
    "\n",
    "attempts_limit = 3\n",
    "\n",
    "# Data pre-processing\n",
    "# allow only 2 decimal places for float dtypes\n",
    "for col in data_df.select_dtypes(float).columns:\n",
    "\tdata_df[col] = data_df[col].round(2)\n",
    "# if the label column is not object, convert it to string\n",
    "if data_df[label_col].dtype != object:\n",
    "\tdata_df[label_col] = data_df[label_col].astype(str)\n",
    "\n",
    "print(f'Model: {model}')\n",
    "context_tokens_limit = 15_000  # lesser values help avoid rate limits\n",
    "\n",
    "print(f'Data rows: {len(data_df)}')\n",
    "data_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B. \tCreating a Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create lot of test data\n",
    "\n",
    "# for each label, get count of rows with that label\n",
    "label_counts = data_df[label_col].value_counts().to_dict()\n",
    "test_df = data_df.copy()\n",
    "test_df.drop(test_df.index, inplace=True)  # remove all rows\n",
    "\n",
    "for label in label_counts:\n",
    "\trows_count = label_counts[label]  # number of rows to create for that label\n",
    "\tsame_label_df = data_df[data_df[label_col] == label].copy()\n",
    "\tnumeric_cols = same_label_df.select_dtypes([int, float]).columns\n",
    "\tnon_numeric_cols = same_label_df.dtypes.index.difference(numeric_cols).difference([label_col])\n",
    "\n",
    "\ttest_rows_count = 10  # rows to create per label\n",
    "\tif rows_count < 40:\n",
    "\t\ttest_rows_count = rows_count * 0.2\n",
    "\t\tif test_rows_count % 1 > 0.5:\n",
    "\t\t\ttest_rows_count += 1\n",
    "\t\ttest_rows_count = int(test_rows_count)\n",
    "\tif test_rows_count < 1 and rows_count >= 1:\n",
    "\t\ttest_rows_count = 1\n",
    "\t\n",
    "\tsize = 2  # number of rows to average at a time\n",
    "\trandom_rows_count = test_rows_count * size  # number of random rows to select\n",
    "\tif random_rows_count > rows_count:\n",
    "\t\trandom_rows_count = rows_count\n",
    "\t# Select 20 random rows\n",
    "\trandom_rows = same_label_df.sample(n=random_rows_count, random_state=random_state).reset_index(drop=True)\n",
    "\n",
    "\tfor i in range(0, random_rows_count, size):\n",
    "\t\tnew_index = len(test_df)\n",
    "\t\ttest_df.loc[new_index] = random_rows.iloc[i]  # copy data of the first row\n",
    "\t\t# test_df.loc[new_index, numeric_cols] = random_rows.loc[i:i+size-1, numeric_cols].mean()  # find averages of 2 rows\n",
    "\t\tfor col in numeric_cols:\n",
    "\t\t\ttest_df.loc[new_index, col] = random_rows.iloc[i:i+size][col].mean()\n",
    "\t\t\tif same_label_df[col].dtype == int:  # if source column has int type, convert the result to int\n",
    "\t\t\t\ttest_df.loc[new_index, col] = int(test_df.loc[new_index, col])\n",
    "\t\tfor col in non_numeric_cols:  # use the most frequent value for non-numeric columns\n",
    "\t\t\tmode_value = random_rows.iloc[i:i+size][col].dropna().mode()\n",
    "\t\t\ttest_df.loc[new_index, col] = mode_value[0] if not mode_value.empty else None\n",
    "\n",
    "\t# count of rows with that label in the test data\n",
    "\tprint(f'{label}: {len(test_df[test_df[label_col] == label])} from {rows_count}')\n",
    "\n",
    "\n",
    "test_df.reset_index(drop=True, inplace=True)\n",
    "print(f'Test data rows: {len(test_df)}')\n",
    "print(f'Test data cols: {len(test_df.columns)}')\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## C. \tDividing data into chunks\n",
    "### Finding best number of chunks to divide the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use any tokenizer to count approximate number of tokens of all the models\n",
    "enc = tiktoken.encoding_for_model('gpt-4')\n",
    "\n",
    "chunk_count = 1  # current number of chunks\n",
    "chunk_size = len(data_df) // chunk_count\n",
    "\n",
    "def get_chunk_text(chunk_index, df=data_df):\n",
    "\tchunk_start = chunk_index * chunk_size\n",
    "\tchunk_end = chunk_start + chunk_size\n",
    "\ttrain_data_chunk = df.iloc[chunk_start:chunk_end].to_csv(index=False).strip()\n",
    "\treturn train_data_chunk\n",
    "\n",
    "while True:\n",
    "\tchunk_size = len(data_df) // chunk_count\n",
    "\ttrain_data_chunk = get_chunk_text(0)\n",
    "\ttoken_count = len(enc.encode(train_data_chunk))\n",
    "\tprint_progress()\n",
    "\tif token_count > 0.85 * context_tokens_limit:\n",
    "\t\tchunk_count += 1  # keep increasing until the chunk size fits the limit\n",
    "\telse:\n",
    "\t\tbreak\n",
    "\n",
    "query_chunk_size = chunk_size * 2\n",
    "print(f'Chunk count: {chunk_count}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## D. \tSummarizing Each Chunk of Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "patterns_prompt_template = '''\n",
    "{train_data_chunk}\n",
    "--\n",
    "Act as an Explainable Machine Learning Model. Don't write code.\n",
    "If there is a bias in the data, highlight it in bold first and how you will handle it.\n",
    "If a small portion of data has unusual patterns or is suspicious, consider it noise or data poisoning,\n",
    "mention it, and ignore it when creating a summary table.\n",
    "\n",
    "Create a table by observing patterns for each label in the dataset.\n",
    "Find exact patterns that separate each label from the rest.\n",
    "Include the column name \"Label ({label_column})\", other columns from the data, and then \"Num_rows\" with the number of rows with that label.\n",
    "Patterns should include be in \"a, b, c\" format for categories or \"min-max [avg]\" format for numbers.\n",
    "Add a \"Comments\" column for each label to write comments about the patterns and any unusual patterns.\n",
    "Write a table between tags <{tag}> and </{tag}>.\n",
    "Ensure only 1 row per label is in the table.\n",
    "The comments column is mandatory.\n",
    "\n",
    "Available Labels: `{available_labels}`\n",
    "'''.strip()\n",
    "\n",
    "attempts_limit = 4\n",
    "tag = 'patterns'\n",
    "comments_col = 'Comments'\n",
    "all_patterns = []\n",
    "\n",
    "def comments_col_exists(df: pd.DataFrame):\n",
    "\tfor col in df.columns:\n",
    "\t\tif comments_col.lower() in col.lower():\n",
    "\t\t\treturn True\n",
    "\treturn False\n",
    "\n",
    "failed_chunks = []\n",
    "chunks_errors = {}\n",
    "\n",
    "error_message = None\n",
    "chunk_result = {}  # 1: False, 2: None, ....\n",
    "patterns_response = None\n",
    "\n",
    "dataset_first_word = dataset_name.split()[0].lower()\n",
    "# enable saving final summary\n",
    "final_summary_filename = os.path.join(data_folder, f'FinalSummary-{model}-{dataset_name}.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_summary_df = None\n",
    "if os.path.exists(final_summary_filename) and os.path.getsize(final_summary_filename):\n",
    "\tfinal_summary_df = pd.read_csv(final_summary_filename)\n",
    "\n",
    "def save_final_summary(df: pd.DataFrame):\n",
    "\tdf.to_csv(final_summary_filename, index=False)\n",
    "\n",
    "def process_chunks():\n",
    "\tglobal chunk_result, failed_chunks, chunks_errors, patterns_response\n",
    "\tfailed_chunks = []\n",
    "\tchunks_errors = {}\n",
    "\tfor chunk_number in range(chunk_count):\n",
    "\t\tif chunk_number % 5 == 0:\n",
    "\t\t\tprint_progress(chunk_number)\n",
    "\t\t# continue if value for chunk_number is already in all_patterns list (use length)\n",
    "\t\tif chunk_result.get(chunk_number, None) != None:\n",
    "\t\t\tprint_progress(chunk_result[chunk_number])\n",
    "\t\t\tcontinue\n",
    "\t\tfor _ in range(attempts_limit):\n",
    "\t\t\ttry:\n",
    "\t\t\t\tpatterns_prompt = patterns_prompt_template.format(\n",
    "\t\t\t\t\tlabel_column=label_col, \n",
    "\t\t\t\t\ttag=tag, train_data_chunk=get_chunk_text(chunk_number),\n",
    "\t\t\t\t\tavailable_labels=', '.join(data_df[label_col].unique()),\n",
    "\t \t\t\t)\n",
    "\t\t\t\tpatterns_response = get_lm_response(patterns_prompt)  # get predictions\n",
    "\t\t\t\tpatterns_response = extract_data(patterns_response, tag=tag)\n",
    "\t\t\t\tsummary_data_df = pd.read_csv(StringIO(patterns_response), sep='|', skiprows=0, skipinitialspace=True)\n",
    "\t\t\t\t# if no column contains \"Comments\", raise error\n",
    "\t\t\t\tif not comments_col_exists(summary_data_df):\n",
    "\t\t\t\t\traise ValueError(f'{comments_col} column is missing')\n",
    "\t\t\t\tall_patterns.append(patterns_response)\n",
    "\t\t\t\tchunk_result[chunk_number] = True\n",
    "\t\t\t\tprint_progress()\n",
    "\t\t\t\terror_message = None\n",
    "\t\t\t\tbreak  # finish the attempt\n",
    "\t\t\texcept Exception as e:\n",
    "\t\t\t\tchunk_result[chunk_number] = None\n",
    "\t\t\t\terror_message = str(e) + patterns_response\n",
    "\t\t\t\t# print(error_message)\n",
    "\t\t\t\tprint_error(e)\n",
    "\t\t# if failed for chunks, add chunk to an array\n",
    "\t\tif error_message:\n",
    "\t\t\tchunk_result[chunk_number] = None\n",
    "\t\t\tprint_progress(f'F{chunk_number} ')\n",
    "\t\t\tfailed_chunks.append(chunk_number)\n",
    "\t\t\tchunks_errors[chunk_number] = error_message\n",
    "\n",
    "if final_summary_df is not None:\n",
    "\tprint('Final Summary already loaded')\n",
    "else:\n",
    "\tprint(f'Chunk count: {chunk_count}')\n",
    "\tprocess_chunks()\n",
    "\n",
    "\tif len(failed_chunks):\n",
    "\t\tprint(f'Failed chunks: {failed_chunks}')\n",
    "\t\tprocess_chunks()\n",
    "\t\tif len(failed_chunks):\n",
    "\t\t\tprint(f'Failed chunks: {failed_chunks}')\n",
    "\t\t\tprint(f'Chunks errors: {chunks_errors}')\n",
    "\t\t\tprint('ERROR: Failed to generate patterns for some chunks')\n",
    "\n",
    "\tdisplay(Markdown(patterns_response))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## E. \tGenerating a Final Summary of All the Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_patterns = all_patterns.copy()\n",
    "patterns_chunk_size = len(summary_patterns) // chunk_count\n",
    "\n",
    "def get_patterns_chunk(chunk_index, data=summary_patterns):\n",
    "\tchunk_start = chunk_index * patterns_chunk_size\n",
    "\tchunk_end = min((chunk_index + 1) * patterns_chunk_size, len(data))\n",
    "\ttext = '\\n---\\n'.join(data[chunk_start:chunk_end])\n",
    "\treturn text.strip()\n",
    "\n",
    "# Finding best chunk size\n",
    "patterns_chunk_count = 1\n",
    "while True:\n",
    "\tpatterns_chunk_size = len(summary_patterns) // patterns_chunk_count\n",
    "\tpatterns = get_patterns_chunk(0)\n",
    "\tpatterns_token_count = len(enc.encode(patterns))\n",
    "\tif patterns_token_count > 0.85 * context_tokens_limit:\n",
    "\t\tpatterns_chunk_count += 1\n",
    "\telse:\n",
    "\t\tbreak\n",
    "print(f'Patterns - chunk count: {patterns_chunk_count}')\n",
    "print(f'Patterns - chunk size: {patterns_chunk_size}')\n",
    "summary_data = None\n",
    "summary_data_df = None\n",
    "summary_data_text = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary_patterns = all_patterns.copy()\n",
    "\n",
    "summary_prompt_template = '''\n",
    "{all_summaries}\n",
    "--\n",
    "Act as an Explainable Machine Learning Model.\n",
    "\n",
    "Each summary above is generated from each chunk of the dataset by finding patterns of the data that separate each label from the rest.\n",
    "Write a table to combine the summaries into a single summary.\n",
    "Include the column name \"Label ({label_column})\", other columns, and \"Num_rows\" with the total number of rows with that label.\n",
    "Add a \"Comments\" column for each label to write comments about the patterns and any unusual patterns.\n",
    "Use the total values using the Num_rows column of each summary.\n",
    "\n",
    "Write a table between tags <{tag}> and </{tag}>.\n",
    "Ensure only 1 row per label is in the table and no extra rows are present.\n",
    "Available Labels: `{available_labels}`\n",
    "\n",
    "Respond with CSV text like: ```csv\n",
    "Label ({label_column}), col1, col2, col3, Num_rows\n",
    "\"1\", a, b, \"c, d, e\", 100\n",
    "\"2\", a2, b2, \"c2, d2, e2\", 200\n",
    "```\n",
    "The CSV must be readable in a Pandas DataFrame.\n",
    "Include quotes for cells with commas. All labels must be strings.\n",
    "CSV must be between tags <{tag}> and </{tag}>.\n",
    "'''.strip()\n",
    "tag = 'patterns'\n",
    "\n",
    "def process_summary_data_df(summary_data):\n",
    "\tglobal summary_data_df\n",
    "\t# load above markdown text to CSV\n",
    "\tsummary_data_df = pd.read_csv(StringIO(summary_data)) #, sep='|', skiprows=0, skipinitialspace=True)  #, index_col=False\n",
    "\tif summary_data_df.index[0] != 0 or summary_data_df.index.__class__.__name__ == 'MultiIndex':\n",
    "\t\tsummary_data_df = pd.read_csv(StringIO(summary_data), index_col=False)\n",
    "\tsummary_data_df.columns = summary_data_df.columns.str.strip()\n",
    "\tcols = summary_data_df.columns\n",
    "\tsummary_data_df = summary_data_df.iloc[:, ~cols.str.contains('^Unnamed')]  # drop unnamed columns\n",
    "\tcols = summary_data_df.columns\n",
    "\t# find first column name that has 'Label' in it\n",
    "\tsummary_label_col = cols[cols.str.contains('Label')][0]\n",
    "\t# convert to string. df label should be stripped\n",
    "\tsummary_data_df[summary_label_col] = summary_data_df[summary_label_col].astype(str).str.strip().str.strip('*')\n",
    "\t# fill Num_rows column with proper values by counting labels taken form Label column of df,\n",
    "\t# and label_col of df_train\n",
    "\tsummary_data_df['Num_rows'] = summary_data_df.apply(lambda row: \n",
    "\t\t\t\t\tdata_df[data_df[label_col] == row[summary_label_col]].shape[0], axis=1)\n",
    "\n",
    "\t# if a row has class as NaN, and all other values as NaN or 0, and comments not None, merge comments to last row\n",
    "\t# and drop that row\n",
    "\tfor index, row in summary_data_df.iterrows():\n",
    "\t\tif row[summary_label_col] == 0 and row.drop(summary_label_col).drop(comments_col).isna().all():  # Values are empty\n",
    "\t\t\tlast_row = summary_data_df.iloc[index - 1]\n",
    "\t\t\tlabel_lower = row[summary_label_col].lower()\n",
    "\t\t\tunknown_label = ('unknown' in label_lower or 'other' in label_lower) \\\n",
    "\t\t\t\tand row[summary_label_col] not in data_df[label_col].unique()\n",
    "\t\t\tif unknown_label and row[comments_col] and last_row[comments_col]:\n",
    "\t\t\t\tlast_row[comments_col] += f'. {row[comments_col]}'  # merge comment with last comment\n",
    "\t\t\tsummary_data_df.drop(index, inplace=True)\n",
    "\tsummary_data_text = summary_data_df.to_csv(index=False).strip()  # convert back summary_data\n",
    "\treturn summary_data_text\n",
    "\n",
    "if final_summary_df is not None:\n",
    "\tsummary_data_df = final_summary_df\n",
    "\tsummary_data_text = summary_data_df.to_csv(index=False).strip()\n",
    "else:\n",
    "\tif len(summary_patterns) == 1:\n",
    "\t\tsummary_data_text = summary_patterns.append('')\n",
    "\n",
    "\twhile len(summary_patterns) > 1:\n",
    "\t\tprint()\n",
    "\t\tfor attempt in range(attempts_limit):\n",
    "\t\t\ttry:\n",
    "\t\t\t\tnew_summary_patterns = []\n",
    "\t\t\t\tpatterns_chunk_count = len(summary_patterns) / patterns_chunk_size\n",
    "\t\t\t\tif patterns_chunk_count % 1:  # decimal value exists\n",
    "\t\t\t\t\tpatterns_chunk_count += 1\n",
    "\t\t\t\tpatterns_chunk_count = int(patterns_chunk_count)\n",
    "\t\t\t\tprint(f'Number of summaries: {len(summary_patterns)}')\n",
    "\t\t\t\tprint(f'Number of chunks: {patterns_chunk_count}')\n",
    "\t\t\t\tfor chunk_number in range(patterns_chunk_count):\n",
    "\t\t\t\t\tif chunk_number % 5 == 0:\n",
    "\t\t\t\t\t\tprint_progress(chunk_number)\n",
    "\t\t\t\t\tall_summaries = get_patterns_chunk(chunk_number)\n",
    "\t\t\t\t\tif not all_summaries:\n",
    "\t\t\t\t\t\tcontinue\n",
    "\t\t\t\t\tpatterns_prompt = summary_prompt_template.format(\n",
    "\t\t\t\t\t\tall_summaries=all_summaries, tag=tag, label_column=label_col,\n",
    "\t\t\t\t\t\tavailable_labels=', '.join(data_df[label_col].unique()),\n",
    "\t\t\t\t\t)\n",
    "\t\t\t\t\tsummary_response = get_lm_response(patterns_prompt)\n",
    "\t\t\t\t\tsummary_data = extract_data(summary_response, tag=tag)\n",
    "\t\t\t\t\tif not summary_data:\n",
    "\t\t\t\t\t\traise ValueError('No data found in summary response')\n",
    "\n",
    "\t\t\t\t\tsummary_data_text = process_summary_data_df(summary_data)\n",
    "\t\t\t\t\tnew_summary_patterns.append(summary_data_text)\n",
    "\t\t\t\t\tprint_progress()\n",
    "\t\t\t\tsummary_patterns = new_summary_patterns\n",
    "\t\t\t\tprint()\n",
    "\t\t\t\tbreak  # finish the attempt\n",
    "\t\t\texcept Exception as e:\n",
    "\t\t\t\tprint(e)\n",
    "\t\t\t\tprint_error(e)\n",
    "\n",
    "\tsave_final_summary(summary_data_df)\n",
    "\n",
    "summary_data_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## F. \tRetrieving Relevant Rows from the Dataset\n",
    "### Fetching data from `df` to answer a query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfquery_prompt_template = '''\n",
    "Data types of the columns:\\n{dtypes_data}\n",
    "\\n ---\n",
    "Summary of the data:\n",
    "{summary_data}\n",
    "\\n ---\n",
    "Test data:\n",
    "{test_df}\n",
    "\\n ---\n",
    "Act as an Explainable Machine Learning Model.\n",
    "\n",
    "Create a query by observing patterns for each label in the dataset and the test data.\n",
    "The query should work on a Python Pandas DataFrame using the df.query() method.\n",
    "Write a query between tags <{tag}> and </{tag}>.\n",
    "I will ask to use the query response to predict the label of the test data.\n",
    "Between tags, don't add any extra text other than query.\n",
    "Ensure the query is short, simple, and concise and fetches only a few similar rows.\n",
    "\n",
    "Example response with query:\n",
    "<{tag}>\n",
    "\t(petal_length > 1.0 and petal_width < 1.0)\n",
    "</{tag}>\n",
    "\n",
    "Columns available for querying: `{available_columns}`\n",
    "\n",
    "It must work with the df.query() method in the Python Pandas library.\n",
    "The query must be short and must use less filters.\n",
    "'''.strip()\n",
    "\n",
    "df_query = None\n",
    "tag = 'dfquery'\n",
    "def get_query_result(df_to_test):\n",
    "\tglobal df_query\n",
    "\tdfquery_prompt = dfquery_prompt_template.format(\n",
    "\t\tsummary_data=summary_data_text, tag=tag, \n",
    "\t\ttest_df=df_to_test.to_csv(index=False).strip(), \n",
    "\t\tavailable_columns=', '.join(data_df.columns), \n",
    "\t\tdtypes_data = str(data_df.dtypes).replace('\\n', ' ; '),\n",
    "\t)\n",
    "\tfor attempt in range(attempts_limit):\n",
    "\t\tdf_query = None\n",
    "\t\ttry:\n",
    "\t\t\tdfquery_response = get_lm_response(dfquery_prompt)  # get query\n",
    "\t\t\tdf_query = extract_data(dfquery_response, tag=tag)\n",
    "\t\t\tdf_query = df_query.replace('\\n', ' ').strip()\n",
    "\t\t\tif len(df_query) > 350:\n",
    "\t\t\t\tprint_error()\n",
    "\t\t\t\tcontinue\n",
    "\n",
    "\t\t\tquery_result = data_df.query(df_query).copy()\n",
    "\t\t\tquery_result.reset_index(drop=True, inplace=True)\n",
    "\t\t\tif not len(query_result):\n",
    "\t\t\t\tdfquery_prompt += f'\\n\\n Last query that returned empty response: `{df_query}`'\n",
    "\t\t\t\tprint_error()\n",
    "\t\t\t\tcontinue\n",
    "\n",
    "\t\t\tif len(query_result) > query_chunk_size:  # allow only `chunk_size` number of rows\n",
    "\t\t\t\tquery_result = query_result.head(query_chunk_size)\n",
    "\t\t\tquery_result = query_result.to_csv(index=False).strip()\n",
    "\t\t\treturn query_result\n",
    "\t\t\t# break\n",
    "\t\texcept Exception as e:\n",
    "\t\t\tprint_error(e)\n",
    "\t\t\tif attempt == attempts_limit-1:\n",
    "\t\t\t\traise Exception('Too many attempts to optimize query')\n",
    "\t\t\tcontinue\n",
    "\n",
    "\traise ValueError('Failed to get result for query')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## G. \tGenerating Classifications and Calculating Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_prompt_template = '''\n",
    "A sample of the dataset with rows similar to the test data:\\n{query_result}\n",
    "\\n ---\n",
    "Summary of the data (with average values in parentheses):\\n{summary_data}\n",
    "\\n ---\n",
    "Test data:\\n{test_data}\n",
    "\\n ---\n",
    "\n",
    "Act as an Explainable Machine Learning Model.\n",
    "\n",
    "Use the above data to make a prediction.\n",
    "Write prediction between tags <{tag}> and </{tag}>, and the reason between <{reason_tag}> and </{reason_tag}>.\n",
    "Between tags, don't add any extra text other than prediction.\n",
    "Write a prediction for the test data.\n",
    "Ignore noise in data and data poisoning attacks, and mention that it is the reason.\n",
    "\n",
    "Sample response:\n",
    "<{tag}> Iris-setosa </{tag}>\n",
    "<{reason_tag}>\n",
    "\t(All rows have SepalLengthCm less than 1.0)\n",
    "</{reason_tag}>\n",
    "\n",
    "Available options: `{available_labels}`\n",
    "\n",
    "Give the same priority to the Summary and Sample of the dataset.\n",
    "'''.strip()\n",
    "\n",
    "tag = 'prediction'\n",
    "reason_tag = 'reason'\n",
    "tag_title = tag.capitalize()\n",
    "\n",
    "prediction = None\n",
    "predictions = []\n",
    "correct_predictions = 0\n",
    "failed_indexes = []\n",
    "error_messages = {}\n",
    "predictions_by_id = {}\n",
    "\n",
    "attempts_limit = 3\n",
    "error_message = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_accuracy(indices = None):\n",
    "\tglobal prediction, predictions, correct_predictions\n",
    "\tglobal failed_indexes, error_messages, error_message\n",
    "\tdata = test_df.iloc[indices] if indices and len(indices) else test_df\n",
    "\tfor index, row in data.iterrows():\n",
    "\t\tif index % 10 == 0:\n",
    "\t\t\tprint_progress(index)\n",
    "\t\tif predictions_by_id.get(index, None) is not None:\n",
    "\t\t\tprint_progress(predictions_by_id[index])\n",
    "\t\t\tcontinue\n",
    "\t\t# error_message = None\n",
    "\t\tfor _ in range(attempts_limit):\n",
    "\t\t\ttry:\n",
    "\t\t\t\ttest_row_df = test_df.iloc[index:index+1].drop(label_col, axis=1)\n",
    "\t\t\t\tquery_result = get_query_result(test_row_df)\n",
    "\t\t\t\tprediction_prompt = prediction_prompt_template.format(\n",
    "\t\t\t\t\ttag=tag, reason_tag=reason_tag, summary_data=summary_data_text, df_query=df_query,\n",
    "\t\t\t\t\ttest_data=test_row_df.to_csv(index=False).strip(), query_result=query_result,\n",
    "\t\t\t\t\tavailable_labels=', '.join(data_df[label_col].unique()),\n",
    "\t\t\t\t)\n",
    "\t\t\t\tprediction = get_lm_response(prediction_prompt)  # get predictions\n",
    "\t\t\t\tif f'<{tag}>' in prediction:\n",
    "\t\t\t\t\tprediction = extract_data(prediction, tag=tag)\n",
    "\t\t\t\telif f'<{tag_title}>' in prediction:\n",
    "\t\t\t\t\tprediction = extract_data(prediction, tag=tag_title)\n",
    "\t\t\t\telse:\n",
    "\t\t\t\t\traise Exception('no tag in response')\n",
    "\t\t\t\tif not prediction:\n",
    "\t\t\t\t\traise Exception('empty prediction')\n",
    "\t\t\t\tprediction = prediction.strip(',').strip()\n",
    "\n",
    "\t\t\t\texpected_label = test_df.iloc[index][label_col]\n",
    "\t\t\t\tis_correct = (prediction == expected_label)\n",
    "\t\t\t\tif not is_correct:\n",
    "\t\t\t\t\tif prediction.lower() == f'class {expected_label}' or \\\n",
    "\t\t \t\t\t\t\tprediction.lower() == f'class-{expected_label}'  or \\\n",
    "\t\t \t\t\t\t\tprediction.lower() == f'classification {expected_label}':\n",
    "\t\t\t\t\t\tprediction = expected_label\n",
    "\t\t\t\t\t\tis_correct = True\n",
    "\t\t\t\tpredictions.append(f'{prediction} expected: {expected_label} correct: {is_correct}')\n",
    "\t\t\t\tif is_correct:\n",
    "\t\t\t\t\tcorrect_predictions += 1\n",
    "\t\t\t\tpredictions_by_id[index] = is_correct\n",
    "\t\t\t\tprint_progress(is_correct)\n",
    "\t\t\t\terror_message = None\n",
    "\n",
    "\t\t\t\tbreak  # finish the attempt\n",
    "\t\t\texcept Exception as e:\n",
    "\t\t\t\tpredictions_by_id[index] = None\n",
    "\t\t\t\t# e = str(e)\n",
    "\t\t\t\t# print('Error:', e)\n",
    "\t\t\t\tprint_error(e)\n",
    "\t\t\t\terror_message = str(e)\n",
    "\t\t\t\tcontinue\n",
    "\t\tif error_message:\n",
    "\t\t\tprint_progress(f'F{index} ')\n",
    "\t\t\tif index not in failed_indexes:\n",
    "\t\t\t\tfailed_indexes.append(index)\n",
    "\t\t\terror_messages[index] = error_message\n",
    "\n",
    "\tprint('\\n correct_predictions:', correct_predictions)\n",
    "\taccuracy = correct_predictions / len(test_df)\n",
    "\treturn accuracy, failed_indexes, error_messages\n",
    "\n",
    "test_rows = len(test_df)\n",
    "train_rows = len(data_df)\n",
    "test_cols = len(test_df.columns)\n",
    "print(f'{model} - {dataset_name} - {train_rows} train rows & {test_rows} test rows * {test_cols} columns')\n",
    "\n",
    "accuracy, failed_indexes, error_messages = get_accuracy()\n",
    "print(f'\\n Accuracy: {accuracy:.2%}')\n",
    "if len(failed_indexes):\n",
    "\tprint(f'Retrying for failed indices: {failed_indexes}')\n",
    "\taccuracy, failed_indexes, error_messages = get_accuracy(failed_indexes)\n",
    "\tif len(failed_indexes):\n",
    "\t\tprint(f'Failed again for indices: {failed_indexes}')\n",
    "\tprint(f'\\n Final Accuracy: {accuracy:.2%}')\n",
    "\n",
    "result_file = 'results.txt'  # Save the result\n",
    "if not os.path.exists(result_file) or not os.stat(result_file).st_size:\n",
    "\twith open(result_file, 'w') as file:\n",
    "\t\tfile.write('Model - Dataset - Train Rows & Test Rows - Accuracy\\n')\n",
    "\n",
    "result_line = f'{model} - {dataset_name} - {train_rows} & {test_rows} * {test_cols} cols - {accuracy:.2%}\\n'\n",
    "with open(result_file, 'a') as file:\n",
    "\tfile.write(result_line)\n",
    "\n",
    "print(result_line)\n",
    "print(f'Prediction count: {len(predictions)}')\n",
    "predictions"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}