-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
11 changed files
with
1,397 additions
and
1,250 deletions.
There are no files selected for viewing
2,067 changes: 1,022 additions & 1,045 deletions
2,067
Cards/Fallacies/Argumentum Fallacies - Taxonomy.csv
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
323 changes: 323 additions & 0 deletions
323
Generation/Converters/Argumentum.AssetConverter/DatasetUpdater/DatasetUpdaterRootConfig.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,323 @@ | ||
| using System.Collections.Generic; | ||
| using System.Threading.Tasks; | ||
| using OpenAI.ObjectModels; | ||
|
|
||
| namespace Argumentum.AssetConverter.DatasetUpdater; | ||
|
|
||
| public class DatasetUpdaterRootConfig | ||
| { | ||
|
|
||
| public async Task Apply(AssetConverterConfig config) | ||
| { | ||
| foreach (var datasetUpdaterConfig in DatasetUpdaterConfigs) | ||
| { | ||
| if (datasetUpdaterConfig.Enabled) | ||
| { | ||
| Logger.LogTitle($"Updating Dataset {datasetUpdaterConfig.SourceDataset}"); | ||
| await datasetUpdaterConfig.Apply(config).ConfigureAwait(false); | ||
| Logger.LogTitle($"Updated Dataset {datasetUpdaterConfig.SourceDataset}"); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private const string PromptsRootPath = @".\DatasetUpdater\Resources\"; | ||
|
|
||
| public List<DatasetUpdaterConfig> DatasetUpdaterConfigs { get; set; } = new List<DatasetUpdaterConfig>() | ||
| { | ||
| new DatasetUpdaterConfig() | ||
| { | ||
| Enabled = false, | ||
| SourceDataset = KnownDataSets.VirtuesTaxonomy, | ||
| FieldsToInclude = new List<string>() | ||
| { | ||
| "path", | ||
| "family_fr", | ||
| "subfamily_fr", | ||
| "subsubfamily_fr", | ||
| "title_fr", | ||
| "description_fr", | ||
| "remark_fr", | ||
| "link_fr" | ||
| }, | ||
| FieldsToUpdate = new List<string>() | ||
| { | ||
| "title_fr", | ||
| "description_fr", | ||
| "remark_fr", | ||
| "link_fr" | ||
| }, | ||
| PrimaryField = "path", | ||
| TargetPath = @".\Target\Datasets\Argumentum Virtues - Taxonomy.csv", | ||
| SystemPromptPath = PromptsRootPath + "VirtuesJsonPromptSystem.txt", | ||
| DialogPrompts = new List<PromptExample>() | ||
| { | ||
| new PromptExample() | ||
| { | ||
| UserPromptPath = PromptsRootPath + "VirtuesJsonPromptSampleUser.json", | ||
| AssistantAnswerPath = PromptsRootPath + "VirtuesJsonPromptSampleAssistant.json" | ||
| } | ||
| }, | ||
| Model = Models.Gpt_3_5_Turbo_1106, | ||
| MaxTokensPerMinute = 70000, | ||
| DivisionMode = DivisionMode.SequentialChunks, | ||
| ChunkSize = 3, | ||
| UseFunctionCalling = false, | ||
| NbMessageCalls = 1, | ||
| SkipChunkNb = 0, | ||
| TakeChunkNb = -1, | ||
| MaxDegreeOfParallelismWebService = 2 | ||
| }, | ||
| new DatasetUpdaterConfig() | ||
| { | ||
| Enabled = false, | ||
| SourceDataset = KnownDataSets.FallaciesTaxonomy, | ||
| FieldsToInclude = new List<string>() | ||
| { | ||
| "path", | ||
| //"Famille", | ||
| //"Sous-Famille", | ||
| //"Soussousfamille", | ||
| "text_fr", | ||
| "desc_fr", | ||
| //"example_fr", | ||
| //"carte", | ||
| //"link_fr" | ||
| "text_en", | ||
| "desc_en", | ||
| //"example_en" | ||
| "link_en" | ||
| }, | ||
| FieldsToUpdate = new List<string>() | ||
| { | ||
| //"path", | ||
| //"text_fr", | ||
| "desc_fr", | ||
| //"example_fr", | ||
| //"link_fr" | ||
| }, | ||
| PrimaryField = "path", | ||
| TargetPath = @".\Target\Datasets\Argumentum Fallacies - Taxonomy.csv", | ||
| SystemPromptPath = PromptsRootPath + "PromptGeneralSystem.txt", | ||
| DialogPrompts = new List<PromptExample>() | ||
| { | ||
| new PromptExample() | ||
| { | ||
| UserPromptPath = PromptsRootPath + "PromptDocumentsLightUser.txt", | ||
| AssistantAnswerPath = PromptsRootPath + "PromptDocumentsAssistant.txt" | ||
| }, | ||
| new PromptExample() | ||
| { | ||
| UserPromptPath = PromptsRootPath + "PromptInstructionsUserDescription.txt", | ||
| AssistantAnswerPath = PromptsRootPath + "PromptInstructionsAssistantDescription.txt" | ||
| } | ||
| }, | ||
| Model = Models.Gpt_4_1106_preview, | ||
| MaxTokensPerMinute = 70000, | ||
| DivisionMode = DivisionMode.PKHierarchicalChar, | ||
| PKHierarchyLevel = 3, | ||
| UseFunctionCalling = true, | ||
| //FunctionName = nameof(RecordsUpdater.UpdateRecord), | ||
| NbMessageCalls = 1, | ||
| SkipChunkNb = 0, | ||
| TakeChunkNb = -1, | ||
| RandomizeChunks = true, | ||
| MaxDegreeOfParallelismWebService = 3, | ||
| CompareMode = true, | ||
| AutoCompare = true, | ||
| AutoCompareField = "text_fr", | ||
| CompareField = "desc_fr", | ||
| MaxGroupItemNb = 20, | ||
| MaxChildren = 12, | ||
| SelectEmptyTargets = false | ||
| }, | ||
| new DatasetUpdaterConfig() | ||
| { | ||
| Enabled = false, | ||
| SourceDataset = KnownDataSets.FallaciesTaxonomy, | ||
| FieldsToInclude = new List<string>() | ||
| { | ||
| "path", | ||
| //"Famille", | ||
| //"Sous-Famille", | ||
| //"Soussousfamille", | ||
| "text_fr", | ||
| "desc_fr", | ||
| "example_fr", | ||
| //"carte", | ||
| //"link_fr" | ||
| "text_en", | ||
| //"desc_en", | ||
| //"example_en", | ||
| //"link_en" | ||
| }, | ||
| FieldsToUpdate = new List<string>() | ||
| { | ||
| //"path", | ||
| //"text_fr", | ||
| //"desc_fr", | ||
| "example_fr", | ||
| //"link_fr" | ||
| }, | ||
| PrimaryField = "path", | ||
| TargetPath = @".\Target\Datasets\Argumentum Fallacies - Taxonomy.csv", | ||
| SystemPromptPath = PromptsRootPath + "PromptGeneralSystem.txt", | ||
| DialogPrompts = new List<PromptExample>() | ||
| { | ||
| new PromptExample() | ||
| { | ||
| UserPromptPath = PromptsRootPath + "PromptDocumentsLightUser.txt", | ||
| AssistantAnswerPath = PromptsRootPath + "PromptDocumentsAssistant.txt" | ||
| }, | ||
| new PromptExample() | ||
| { | ||
| UserPromptPath = PromptsRootPath + "PromptInstructionsLightUserExamples.txt", | ||
| AssistantAnswerPath = PromptsRootPath + "PromptInstructionsLightAssistantExamples.txt" | ||
| } | ||
| }, | ||
| Model = Models.Gpt_4_0125_preview, | ||
| MaxTokensPerMinute = 70000, | ||
| DivisionMode = DivisionMode.PKHierarchicalChar, | ||
| PKHierarchyLevel = 3, | ||
| UseFunctionCalling = true, | ||
| //FunctionName = nameof(RecordsUpdater.UpdateRecord), | ||
| NbMessageCalls = 1, | ||
| SkipChunkNb = 0, | ||
| TakeChunkNb = 10, | ||
| SelectEmptyTargets = true, | ||
| RandomizeChunks = false, | ||
| MaxDegreeOfParallelismWebService = 3, | ||
| CompareMode = false, | ||
| AutoCompare = true, | ||
| AutoCompareField = "text_fr", | ||
| CompareField = "example_fr", | ||
| MaxGroupItemNb = 30, | ||
| WriteOneTargetFileByField = true, | ||
| MaxChildren = 12 | ||
| }, | ||
| new DatasetUpdaterConfig() | ||
| { | ||
| Enabled = false, | ||
| SourceDataset = KnownDataSets.FallaciesTaxonomy, | ||
| FieldsToInclude = new List<string>() | ||
| { | ||
| "path", | ||
| //"Famille", | ||
| //"Sous-Famille", | ||
| //"Soussousfamille", | ||
| "text_fr", | ||
| "desc_fr", | ||
| "example_fr", | ||
| //"carte", | ||
| "link_fr", | ||
| "text_en", | ||
| "desc_en", | ||
| "example_en", | ||
| "link_en" | ||
| }, | ||
| FieldsToUpdate = new List<string>() | ||
| { | ||
| //"path", | ||
| //"text_fr", | ||
| "desc_fr", | ||
| "example_fr", | ||
| //"link_fr", | ||
| "text_en", | ||
| "desc_en", | ||
| "example_en", | ||
| }, | ||
| PrimaryField = "path", | ||
| TargetPath = @".\Target\Datasets\Argumentum Fallacies - Taxonomy.csv", | ||
| SystemPromptPath = PromptsRootPath + "PromptGeneralSystem.txt", | ||
| DialogPrompts = new List<PromptExample>() | ||
| { | ||
| new PromptExample() | ||
| { | ||
| UserPromptPath = PromptsRootPath + "PromptTranslateFrEnInstructionsUser.txt", | ||
| AssistantAnswerPath = PromptsRootPath + "PromptTranslateFrEnInstructionsAssistant.txt" | ||
| } | ||
| }, | ||
| Model = Models.Gpt_4_0125_preview, | ||
| MaxTokensPerMinute = 70000, | ||
| DivisionMode = DivisionMode.PKHierarchicalChar, | ||
| PKHierarchyLevel = 3, | ||
| UseFunctionCalling = true, | ||
| //FunctionName = nameof(RecordsUpdater.UpdateRecord), | ||
| NbMessageCalls = 1, | ||
| SkipChunkNb = 0, | ||
| TakeChunkNb = -1, | ||
| SelectEmptyTargets = true, | ||
| RandomizeChunks = false, | ||
| MaxDegreeOfParallelismWebService = 3, | ||
| CompareMode = false, | ||
| AutoCompare = true, | ||
| AutoCompareField = "text_fr", | ||
| CompareField = "example_fr", | ||
| MaxGroupItemNb = 30, | ||
| WriteOneTargetFileByField = true, | ||
| MaxChildren = 12 | ||
| }, | ||
| new DatasetUpdaterConfig() | ||
| { | ||
| Enabled = true, | ||
| SourceDataset = KnownDataSets.FallaciesTaxonomy, | ||
| FieldsToInclude = new List<string>() | ||
| { | ||
| "path", | ||
| //"Famille", | ||
| //"Sous-Famille", | ||
| //"Soussousfamille", | ||
| "text_fr", | ||
| "desc_fr", | ||
| "example_fr", | ||
| //"carte", | ||
| "link_fr", | ||
| "text_en", | ||
| "desc_en", | ||
| "example_en", | ||
| "link_en", | ||
| "text_ru", | ||
| "desc_ru", | ||
| "example_ru", | ||
| "link_ru" | ||
| }, | ||
| FieldsToUpdate = new List<string>() | ||
| { | ||
| "text_ru", | ||
| "desc_ru", | ||
| "example_ru" | ||
| }, | ||
| PrimaryField = "path", | ||
| TargetPath = @".\Target\Datasets\Argumentum Fallacies - Taxonomy.csv", | ||
| SystemPromptPath = PromptsRootPath + "PromptGeneralSystem.txt", | ||
| DialogPrompts = new List<PromptExample>() | ||
| { | ||
| new PromptExample() | ||
| { | ||
| UserPromptPath = PromptsRootPath + "PromptTranslateRuInstructionsUser.txt", | ||
| AssistantAnswerPath = PromptsRootPath + "PromptTranslateRuInstructionsAssistant.txt" | ||
| } | ||
| }, | ||
| Model = Models.Gpt_4_0125_preview, | ||
| MaxTokensPerMinute = 70000, | ||
| DivisionMode = DivisionMode.SequentialChunks, | ||
| PKHierarchyLevel = 3, | ||
| UseFunctionCalling = true, | ||
| //FunctionName = nameof(RecordsUpdater.UpdateRecord), | ||
| NbMessageCalls = 1, | ||
| SkipChunkNb = 0, | ||
| TakeChunkNb = 5, | ||
| ChunkSize = 8, | ||
| SelectEmptyTargets = true, | ||
| RandomizeChunks = false, | ||
| MaxDegreeOfParallelismWebService = 3, | ||
| CompareMode = false, | ||
| AutoCompare = true, | ||
| AutoCompareField = "text_fr", | ||
| CompareField = "example_fr", | ||
| MaxGroupItemNb = 12, | ||
| WriteOneTargetFileByField = false, | ||
| MaxChildren = 8 | ||
| } | ||
|
|
||
| }; | ||
| } |
2 changes: 1 addition & 1 deletion
2
...ion/Converters/Argumentum.AssetConverter/DatasetUpdater/Resources/PromptGeneralSystem.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1 @@ | ||
| Tu es un expert en rhétorique et en linguistique, doté d'une maîtrise polyglotte. Ta spécialité est l'analyse précise et la correction de documents complexes. Tu possèdes une compréhension approfondie des structures et des nuances linguistiques, te permettant d'apporter des améliorations ciblées et pertinentes à divers types de documents thématiques. | ||
| Tu es un expert en rhétorique, en linguistique et en traduction, doté d'une maîtrise polyglotte. Ta spécialité est l'analyse précise, la traduction et la correction de documents complexes. Tu possèdes une compréhension approfondie des structures et des nuances linguistiques, te permettant d'apporter des améliorations ciblées et pertinentes à divers types de documents thématiques. |
2 changes: 2 additions & 0 deletions
2
...ntum.AssetConverter/DatasetUpdater/Resources/PromptTranslateFrEnInstructionsAssistant.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,2 @@ | ||
| J'ai bien pris note des instructions détaillées et je me concentrerai sur la traduction vers l'anglais, en identifiant les entités pour lesquelles elle est nécessaire, et en proposant de les remplir de la façon la plus adéquate. Je m'assurerai bien de ne pas citer l'argument dans l'exemple qui doit se suffire à lui-même. Je ne chercherai pas à répondre textuellement au prochain message que vous allez me soumettre, mais j'utiliserai plutôt la fonctionnalité de function-calling de l'API OpenAI pour proposer chacune des mes modifications grâce aux fonctions que tu m'as mises à disposition. | ||
| Prêt à recevoir les données pour la mise à jour. |
Oops, something went wrong.