Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize and fix prompt generation #6

Merged
merged 5 commits into from
Oct 24, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Optimize and fix quality of prompt generation
  • Loading branch information
mzukowski-reef committed Sep 23, 2024
commit 664c7dfb08f0dae6eefbcf76e0a58ad9330a4a52
16 changes: 9 additions & 7 deletions src/compute_horde_prompt_gen/prompt.py
Original file line number Diff line number Diff line change
@@ -4,17 +4,19 @@

class PromptGeneratingPrompt:
def random_select(self, arr: list[str], num: int = 5) -> str:
random.shuffle(arr)
return ", ".join(arr[:num]) + ", etc"
return random.choices(arr, k=num)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
return random.choices(arr, k=num)
return random.sample(arr, k=num)

I think we should not pick the same thing twice.


def generate_prompt(self, short=True) -> str:
themes = self.random_select(THEMES, num=3)
def random_select_str(self, arr: list[str], num: int = 5) -> str:
choices = self.random_select(arr, num)
return ", ".join(choices) + ", etc"

def generate_prompt(self, short=True) -> str:
if short:
theme = self.random_select(THEMES, num=1)[0]
return (
f"Generate a list of 10 questions or instruct tasks related to the themes of {themes}. "
f"Output each prompt on a new line without any extra commentary or special characters."
f"{theme}"
)
themes = self.random_select_str(arr, num=3)

relevance_level = random.randint(5, 20)
complexity_level = random.randint(5, 20)
@@ -34,5 +36,5 @@ def generate_prompt(self, short=True) -> str:

def generate_role(self, short=True) -> str:
if short:
return "You are a prompt engineer tasked with prompts of varying complexity to test the capabilities of a new language model."
return "You are my friend helping me study. I say theme and you ask me question about it. Quiestions have to be short but open. Write only question and nothing more."
return "You are a prompt engineer tasked with prompts of varying complexity to test the capabilities of a new language model. For each prompt, consider what aspect of the language model's capabilities it is designed to test and ensure that the set of prompts covers a broad spectrum of potential use cases for the language model. Only output the prompts, one per line without any extra commentary. Do not use any special characters or formatting, numbering or styling in the output."
10 changes: 5 additions & 5 deletions src/compute_horde_prompt_gen/run.py
Original file line number Diff line number Diff line change
@@ -38,22 +38,22 @@ def generate_prompts(
)

seconds_taken = (datetime.datetime.now() - start_ts).total_seconds()
log.info(f"{i=} generation took {seconds_taken:.2f}s")

new_prompts = []
for j, sequence in enumerate(sequences):
generated_prompts = parse_output(sequence)
log.debug(f"{i=} sequence={j} {generated_prompts=} from {sequence=}")

log.info(f"{i=} sequence={j} generated {len(generated_prompts)} prompts")
new_prompts.extend(generated_prompts)

# check_prompts_quality(new_prompts)

# remove any duplicates
new_prompts = list(set(new_prompts))

log.info(f"{i=} generation took {seconds_taken:.2f}s; generated {len(new_prompts)} prompts")
if total_prompts - len(new_prompts) < 0:
# one might want to optimize here and save additional prompts for next batch,
# but it is so parametrized that it produces on average additional 10 prompts
# so to fill 240 we would nedd 24 runs - each 10s to produce additional batch
# for free - saving 10s - so about 4% gain - not worth it :)
new_prompts = new_prompts[:total_prompts]

total_prompts -= len(new_prompts)
24 changes: 16 additions & 8 deletions src/compute_horde_prompt_gen/utils.py
Original file line number Diff line number Diff line change
@@ -8,22 +8,30 @@

def clean_line(line: str) -> str:
line = line.strip()
head, sep, tail = line.partition('<|')
if head:
line = head.strip()
else:
# if we started with a tag we assume that inside we find our prompt
line = tail.partition('|>')[2].partition('<|')[0].strip()
# remove list numbering if present
line = re.sub(r"^\s*\d+\.?\s*", "", line)
# strip quotations
line = line.strip('"\'')
return line


def parse_output(output: str) -> list[str]:
# split into lines and clean them
lines = output.split("\n")
lines = [clean_line(line) for line in lines]

# filter out null lines or prompts that are too short or long
lines = [line for line in lines if (len(line) > 10 and len(line) < 300)]

# skip first line as that's frequently broken (i.e. "Here are the prompts:")
# skip last line as it might not be comletely generated
return lines[1:-1]
for line in lines:
clean_line = clean_line(line)
# we skip if line is too short or too long and not ends with ?
# in most cases it would be just first line
if len(clean_line) > 10 and len(clean_line) < 300 and line.endswith('?'):
return [line]

return []


def check_prompts_quality(prompts: list[str]):