mono_corpus.py

from tkinter import *
from tkinter import messagebox
from tkinter.filedialog import askopenfilename
from tkinter import scrolledtext
from rake_nltk import Rake
import yake
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
import spacy
import Extraction as fcns
from tkinter import ttk
from tkinter.filedialog import askopenfilenames
import os
import matplotlib.pyplot as plt
from matplotlib.backends.backend_tkagg  import FigureCanvasTkAgg

import pstats
from matplotlib.pyplot import switch_backend 
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
import regex as re
import string
import numpy as np
import nltk.data
import re
from nltk.stem import WordNetLemmatizer
from nltk import word_tokenize, sent_tokenize, pos_tag
from sklearn.feature_extraction.text import TfidfVectorizer
import math
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.collocations import BigramAssocMeasures, BigramCollocationFinder, QuadgramAssocMeasures
from nltk.collocations import TrigramAssocMeasures, TrigramCollocationFinder, QuadgramCollocationFinder
from tkinter import filedialog
from tkinter.filedialog import askdirectory
import ClassNettoyage as cn


fenetre = Tk()
fenetre.geometry('1350x680+0+0')
fenetre.title('Interface Extraction des termes à partir d\'un seul CORPUS')
fenetre.configure(bg='light blue')

class App(Frame):
    def __init__(self, master=None):
        super().__init__(master)
        self.master = master
        self.counter = 0
        self.create_widgets()

    def create_widgets(self):
        #self.button = Button(self.master, text="Cliquez ici", command=self.increment_counter)
        #self.button.pack()
        self.label = Label(self.master, text="Compteur : 0")
        self.label.pack()

    def increment_counter(self):
        self.counter += 1
        self.label.config(text="Compteur : {}".format(self.counter))

#root = Tk()
app = App(master=fenetre)
#app.mainloop()

def nettoyer(texte):
    # Creer l'objet et nettoyer
    net = cn.Nettoyage(texte, "Ponctuation.txt", "Stopword_alir3z4.txt") 
    net.nettoyer()

    return net.chaine_net

def extraire_termes_rake(contenus):
    ponctuation = {'.', ',', ';', ':', '-', '_', '(', ')', '\'', '"', '{', '}', ']', '[', '/', '\\', '|', '([' '])',
                   '!', '?', '@', '#', '$', '%', '^', '&', '*', '+', '=', '*', '<', '>',
                   '),', ').', '.)', ').', ')?', ');'}
    
    # Créer un extracteur 
    extracteur = Rake(language='english', include_repeated_phrases=False, max_length=4,
                      stopwords=set(open('Stopword_alir3z4.txt', encoding="utf8").read().split()),
                      punctuations=ponctuation)

    # Parcourir les contenus de fichiers

    termes_par_texte = [] 

    for contenu in contenus:
        extracteur.extract_keywords_from_text(contenu)
        termes_importants = extracteur.get_ranked_phrases_with_scores()[:70]
        termes = [terme[1] for terme in termes_importants] 
        termes_par_texte.append(termes)
    
    return termes_par_texte

def extraire_termes_yake(contenus):

    termes_par_texte = []
    kw_extractor = yake.KeywordExtractor(top=70)

    for contenu in contenus:
        keywords = kw_extractor.extract_keywords(contenu)
        termes = [terme[0] for terme in keywords]
        termes_par_texte.append(termes)

    return termes_par_texte

def extraire_termes_TfIdf(contenus):

    termes_par_texte = []
    for contenu in contenus:
        documents = [
            contenu,
            "Home gardens key to improved nutritional well-being Malnutrition not only results in increased mortality and health problems including infectious diseases, mental retardation and blindness, it is also responsible for loss of human capital and work productivity. About 39 percent of Lao PDR's population is below the national income poverty line and 22 percent are food insecure. The average per capita annual income of US$370 in 1997 instead of increasing towards the targeted US$500 in 2000, declined to US$350 and then further to US$331 by 2003 as a result of inflation. I m p r o v e d nutritional standards lead to improved health, well-being and development o p p o r t u n i t i e s . Evidence from Asian countries, particularly Viet Nam4 and some other countries5 in the region, shows that home gardens in combination with nutrition education can make a highly effective contribution towards nutrition improvement among rural poor households. Aware of the negative impact of high malnutrition levels on the national development potential, and in keeping with its commitment at the 1992 International Conference on Nutrition (ICN) and to the UN Millennium Development Goals (MDGs), the Government of Lao PDR is giving priority to diversifying food consumption to ensure a more balanced diet to its people. As this objective could not be achieved from local expertise and resources alone, the Government requested assistance from FAO. Under the US$332 000 Technical Cooperation Project (TCP) Promotion of home gardens for i m p r o v e d nutritional well-being, TCP/LAO/2902 (A), FAO collaborated with the Government of Lao PDR to promote home gardens for improving nutritional well-being of rural communities in Lao PDR. The pilot project, which commenced operations in February 2003 and concluded in August 2004, has developed and fine-tuned a suitable approach for household food security and nutrition improvement capable of replication at national level. The project included a creative participatory planning and implementation process involving local communities, district-level authorities as well as technical experts and policy makers in the Department of Agriculture (DOA). It has provided an integrated package of home gardening inputs and nutrition education to target households and communities in four villages6 of Vientiane and Bolikhamxay Provinces as well as in Vientiane Municipality. A total of 204 home gardens and four community gardens have been established. The resulting diversified diet is expected to reduce malnutrition and improve health, especially of under-five-year-old children and women of reproductive age. The project also promoted 59 micronutrient-rich foods through the home gardening programme. Another project objective was additional income generation for rural families, not only through direct sale of home produce but also from indirect savings as a result of reduced health care expenditure. The project was implemented in close collaboration with the Ministry o f Health (MOH), Ministry of Education (MOE), Lao Women's Union (LWU) and international non-governmental organizations (NGOs).",
            "Home gardens key to improved nutritional well-being A significant outcome of the project was the development of provincial, district and community-level capacities for implementation and management of home gardening and nutrition improvement programmes. This was done through training of trainers (TOTs) and in-service training (IST) by FAO experts on key modules of nutrition, horticulture, small livestock and fisheries for provincial and district officials of the Department of Agriculture (DOA). A post-project evaluation found increased production of vegetables, fruits, poultry and fish among the target households and awareness created for greater consumption of home-grown produce. Comparison with baseline estimates found that moderate and severe undernutrition rates for under-five-year-old children had declined from 23 to 15.9 percent and 9.5 to 2.3 percent respectively."
            "The objective of TCP/LAO/2902 (A) pilot project Promotion of Hone Gardens for Improved Nutritional Well-Being was to develop a model for household nutrition garden production, including small livestock and aquaculture. Covering a population of 1 000 people from about 200 households in four villages, the project especially targeted families with children less than five years old. The overall project objective was to reduce malnutrition and improve nutritional well-being of the Lao population through increased production and consumption of nutritious food with special emphasis on a micronutrient-rich diet. The immediate objective was to develop a nationally replicable model for increased and diversified household food production and consumption by rural families, in combination with nutrition education. The project had the following specific objectives"
            "Increase the amount and variety of nutritious food for project households with special emphasis on food rich in micronutrients. Carry out an intensive public nutrition education campaign aimed at improving and diversifying family food consumption with special emphasis on children under five years old and women of reproductive age. Establish a network for collaboration between the Ministry of Agriculture and Forestry (MAF), Ministry of Health (MOH), Special Programme for Food Security (SPFS) team, FAO and relevant development partners. Evaluate the results and impact of the project and develop model home gardens for improved nutritional well-being and suitable for national implementation."
        ]
        vectorizer = TfidfVectorizer()
        X = vectorizer.fit_transform(documents)
        feature_names = vectorizer.get_feature_names_out()
        scores = X.sum(axis=0).A1
        termes_importants = [feature_names[i] for i in scores.argsort()[:70]]
        termes_par_texte.append(termes_importants)        
        

    return termes_par_texte

def extraction_methode(contenus,nlp):

    #nlp = spacy.load('en_core_web_lg') 

    # Récupérer le texte de la zone_texte1
    termes_par_texte = []
    i=0
    for contenu in contenus:
        i=i+1
        doc = nlp(contenu)
        #Decouper le texte en phrases
        sentences = list(doc.sents)
    
        extraction = fcns.Extraction(sentences)

        extraction.extractTerms(70)
        app.increment_counter()
        print(f"termes {i} extraits")
        termes = extraction.getTerms()

        termes_par_texte.append(termes)

    return termes_par_texte

def extraction_nmethode(repertoire,rep):

    path = repertoire   
    all_files = os.listdir(path+"docsutf8")
    all_keys = os.listdir(path + "keys")

    all_documents = []
    all_keys = []
    all_files_names = []


    # Reading the files and storing them in a list.
    for i, fname in enumerate(all_files):
        with open(path+'docsutf8/'+fname, encoding="utf8") as f:
            lines = f.readlines()
        key_name = fname[:-4]
        with open(path+'keys/'+key_name+'.key', encoding="utf8") as f:
            k = f.readlines()
        all_text = ' '.join(lines)
        keyss = ' '.join(k)
        all_documents.append(all_text)
        all_keys.append(keyss.split("n"))
        all_files_names.append(key_name)

        print(f"boucle 1 {i+1} fois")


    def preprocess_text(text):
        """
        It removes numbers, http, punctuation, converts to lower case, and removes short words.
    
        :param text: the text to be preprocessed
        """
        text = remove_numbers(text)
        text = remove_http(text)
        text = remove_punctuation(text)
        text = convert_to_lower(text)
        text = remove_short_words(text)
    
        return text
    def convert_to_lower(text):
        """
        It takes a string as input and returns a lowercase version of that string
    
        :param text: The text to be converted to lowercase
        :return: The text is being returned in lower case.
        """
        return text.lower()
    def remove_numbers(text):
        """
        It takes a string as input, removes all numbers from it, and returns the resulting string
    
        :param text: The text that you want to remove numbers from
        :return: The text is being returned without any numbers.
        """
        text = re.sub(r'[0-9]+' , '', text)
        return text
    def remove_http(text):
        """
        It takes a string as input, and returns a string with all the URLs removed
    
        :param text: The text to be cleaned
        :return: A string with the http removed.
        """
        text = re.sub("https?://t.co/[A-Za-z0-9]*", ' ', text)
        return text
    def remove_short_words(text):
        """
        It takes a string as input, and returns a string with all words of length 1 or 2 removed
    
        :param text: The text that you want to clean up
        :return: The text is being returned.
        """
        text = re.sub(r'bw{1,2}b', '', text)
        return text
    def remove_punctuation(text):
        """
        It takes a string as input, and returns a string with all punctuation removed
    
        :param text: the text to be processed
        """
        text = text.replace("-", " ")
        punctuations = '''!()[]{};«№»:'",`./?@=#$-(%^)+&[*_]~'''
        no_punctuation = "" 
        for char in text:
            if char not in punctuations:
                no_punctuation = no_punctuation + char
        return no_punctuation
    def remove_white_space(text):
        """
        It removes white space from the beginning and end of a string
    
        :param text: The text to be cleaned
        :return: The text is being returned.
        """
        text = text.strip()
        return text

    #function that returns list of sipmle terms in the doc
    def toknizing_termesSimple(text, stop_words):
    
        tokens = word_tokenize(text)
        ## Remove Stopwords from tokens
        result = [i for i in tokens if not i in stop_words]
        return result    

    #function that returns list of stop wordin in the doc
    def toknizing_stopwords(text, stop_words):
        #stop_words = set(stopwords.words('english')) 
        tokens = word_tokenize(text)
        ## Remove Stopwords from tokens
        result = [i for i in tokens if i in stop_words]
        return result

    def cal_n(term, N):
        """
        It counts the number of documents a term appears in
    
        :param term: the term you want to calculate the frequency of
        :param N: number of documents
        :return: The number of times the term appears in the corpus.
        """
        n = 0
        for i in range(N):
            found = False
            for j in range(len(termes[i])):
                if str(term) == str(termes[i][j]):
                    found = True
            if found:
                n = n + 1
        return(n)
    def cal_tf(term, doc, N):
        """
        It takes a term, a document, and the number of documents in the corpus, and returns the term
        frequency of the term in the document
    
        :param term: the term we're looking for
        :param doc: the document number
        :param N: number of documents
        :return: The number of times a term appears in a document.
        """
        tf = 0
        len_doc = len(termes[doc])
        for j in range(len_doc):
            if str(term) == str(termes[doc][j]):
                tf = tf + 1
        return(tf)
    def cal_pond(N, n, tf, len_doc, moy_len_doc):
        """
        The function takes the number of documents in the corpus, the number of documents containing the
        term, the term frequency in the document, the length of the document, and the average length of
        documents in the corpus. It returns the TF-IDF score for the term in the document(more so ponderation)
    
        :param N: total number of documents in the collection
        :param n: number of documents containing the term
        :param tf: term frequency
        :param len_doc: the length of the document
        :param moy_len_doc: the average length of a document in the corpus
        :return: the pondered score of a term in a document.
        """
        lengs = 1.5 * (len_doc / moy_len_doc)
        num1 = tf/(tf + 0.5 + lengs)
        num2 = math.log((N + 0.5)/(n+1)) / math.log(N + 1)
        pond = 0.4 + 0.6 * num1 * num2
        return(pond)
    def cal_freq_corpus(term):
        """
        It counts the number of times a term appears in the corpus
    
        :param term: the term we want to calculate the frequency of
        :return: The number of times a term appears in the corpus.
        """
        f = 0
        for i in range(N):
            for j in range(len(termes[i])):
                if str(term) == str(termes[i][j]):
                    f = f + 1
        return(f)
    def cal_proba(term):
        """
        > The function `cal_proba` takes a term as input and returns the probability of that term in the
        corpus
    
        :param term: the term you want to calculate the probability of
        :return: The probability of a term in the corpus.
        """
        proba = cal_freq_corpus(term) / num_words_corpus
        return(proba)
    def cal_p2(term1, term2):
        """
        It counts the number of times a 2-gramm appears in the corpus
    
        :param term1: the first term
        :param term2: the term that we want to calculate the probability of
        :return: The number of times the two terms appear together in the same document.
        """
        p = 0
        for i in range(N):
            for j in range(len(termes[i])-1):
                k = j+1
                if (term1 == termes[i][j]) and (term2 == termes[i][k]):
                    p = p + 1
        return(p)
    def cal_p3(term1, term2, term3):
    
        p = 0
        for i in range(N):
            for j in range(len(termes[i])-2):
                k = j + 1
                l = j + 2
                if (term1 == termes[i][j]) and (term2 == termes[i][k]) and (term3 == termes[i][l]):
                    p = p + 1
        return(p)
    def cal_PMI(term1, term2):
        """
        The function takes two terms as input and returns the PMI of the two terms.
    
        :param term1: the first term
        :param term2: the word you want to find the PMI of
        :return: The PMI of the two terms.
        """
        num1 = cal_p2(term1, term2) * N #um_words_corpus
        num2 = cal_freq_corpus(term1) * cal_freq_corpus(term2)
        PMI = math.log(num1 / num2)
        return(PMI)
    def lang_simple(line):
        line = line.split(" ")
        lang = 0
        for i in range(len(line)):
            if line[i] in simplelist:
                lang += 1
        return lang
    def som_simple(line):
        somme = 0
        line = line.split(" ")
        for i in range(len(line)):
            if line[i] in simplelist:
                j = 0
                while True:
                    p = psimplelist[j][1]
                    j += 1
                    if line[i] == psimplelist[j-1][0]:
                        #print(psimplelist[j-1][0])
                        break
                somme = somme + p
        return somme
    def nbr_doc(line):
        n = 0
        line = line.split(" ")
        if len(line) == 2:
            for i in range(N):
                found = False
                for j in range(len(termes[i])-1):
                    if (line[0] == termes[i][j]) and (line[1] == termes[i][j+1]):
                        found = True
                if found:
                    n = n + 1
        elif len(line) == 3:
            for i in range(N):
                found = False
                for j in range(len(termes[i])-2):
                    if (line[0] == termes[i][j]) and (line[1] == termes[i][j+1]) and (line[2] == termes[i][j+2]):
                        found = True
                if found:
                    n = n + 1
        elif len(line) == 4:
            for i in range(N):
                found = False
                for j in range(len(termes[i])-3):
                    if (line[0] == termes[i][j]) and (line[1] == termes[i][j+1]) and (line[2] == termes[i][j+2]) and (line[3] == termes[i][j+3]):
                        found = True
                if found:
                    n = n + 1
        return n
    def tf_comp(line, doc):
        n = 0
        line = line.split(" ")
        if len(line) == 2:
            for j in range(len(termes[doc])-1):
                if (line[0] == termes[doc][j]) and (line[1] == termes[doc][j+1]): 
                    n = n + 1
        elif len(line) == 3:
            for j in range(len(termes[doc])-2):
                if (line[0] == termes[doc][j]) and (line[1] == termes[doc][j+1]) and (line[2] == termes[doc][j+2]): 
                    n = n + 1
        elif len(line) == 4:
            for j in range(len(termes[doc])-3):
                if (line[0] == termes[doc][j]) and (line[1] == termes[doc][j+1]) and (line[2] == termes[doc][j+2]) and (line[3] == termes[doc][j+3]):
                    n = n + 1
        return n
    def pond_comp(l, p, som):
        num1 = 1 - (1/(l+1))
        num2 = (1/(l+1)) * som
        pond = num1 + p + num2
        return pond

    def creerRepertoire(path):
        if not os.path.exists(path):
            os.mkdir(path)

    _log2 = lambda x: math.log(x, 2.0)

    N = len(all_documents)
    stop_words = set(stopwords.words('english')) 
    termes = []

    # Preprocessing the text and tokenizing it.
    s = 0

    for i in range(N):
        all_documents[i] = preprocess_text(all_documents[i])
        termes.append(word_tokenize(all_documents[i]))
        len_doc = len(termes[i])
        s = s + len_doc
        moy_len_doc = s / N
        #print(moy_len_doc)

        print(f"boucle preprocess {i+1} fois")


    # Counting the number of words in the corpus.
    num_words_corpus = 0
    for i in range(N):
        len_doc = len(termes[i])
        for j in range(len_doc):
            num_words_corpus = num_words_corpus + 1
    #print(num_words_corpus)

    for i in range(N):
        # Tokenizing the terms and stopwords.
        termes_simple = toknizing_termesSimple(all_documents[i], stop_words)
        stopwords_simple = toknizing_stopwords(all_documents[i], stop_words)

        termes_simple = list(dict.fromkeys(termes_simple))
        stopwords_simple = list(dict.fromkeys(stopwords_simple))

        #print(termes_simple)
        #print("\n")
        #print(stopwords)
        #print("\n")
        #chemin = rep + "Inspec\\terms\\newmethod\\termes\%s.txt"
        #open (chemin%(all_files_names[i]), "w")
        # Calculating the term frequency & ponderation of each term in each document.
        #file_simple = open((rep+"Inspec\\terms\\newmethod\\termes\%s.txt")%(all_files_names[i]), "w", encoding="utf8")

        creerRepertoire(rep + "Methode Harrathi")
        creerRepertoire(rep + "Methode Harrathi/termes")
        creerRepertoire(rep + "Methode Harrathi/ptermes")

        file_simple = open((rep + "Methode Harrathi\\termes\%s.txt") % (all_files_names[i]), "w", encoding="utf8")
        file_psimple = open((rep+"Methode Harrathi\ptermes\%s.txt")%(all_files_names[i]), "w", encoding="utf8")

        for x1 in termes_simple:
            n = cal_n(x1, N)
            tf = cal_tf(x1, i, N)
            len_doc = len(termes[i])
            p = cal_pond(N, n, tf, len_doc, moy_len_doc)
            psimple = cal_proba(x1)

            chaine = x1 + " , " + str(p) + "\n"
            file_simple.write(chaine)

            chaine = x1 + " , " + str(psimple) + "\n"
            file_psimple.write(chaine)

        file_simple.close()
        file_psimple.close()

        creerRepertoire(rep + "Methode Harrathi/stopwords")
        creerRepertoire(rep + "Methode Harrathi/pstopwords")


        # Calculating the frequency & ponderation of each stopword in each document.
        file_stopwords = open((rep+"Methode Harrathi\stopwords\%s.txt")%(all_files_names[i]), "w")
        file_pstopwords = open((rep+"Methode Harrathi\pstopwords\%s.txt")%(all_files_names[i]), "w")
        for x2 in stopwords_simple:
            n = cal_n(x2, N)
            tf = cal_tf(x2, i, N)
            len_doc = len(termes[i])
            p = cal_pond(N, n, tf, len_doc, moy_len_doc)
            pstopword = cal_proba(x2)

            chaine = x2 + " , " + str(p) + "\n"
            file_stopwords.write(chaine)

            chaine = x2 + " , " + str(pstopword) + "\n"
            file_pstopwords.write(chaine)

        file_stopwords.close()
        file_pstopwords.close()

        print(f"boucle 3 {i+1} fois")

    for i in range(N):
    
        file_psimple = open((rep+"Methode Harrathi\ptermes\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        num_file_psimple = sum(1 for line in file_psimple)
        file_psimple.close()
        file_psimple = open((rep+"Methode Harrathi\\ptermes\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        psimplelist = []
        for j in range(0,num_file_psimple):
            line = file_psimple.readline().strip().split(" , ")
            line[1] = np.double(line[1])
            psimplelist.append(line)
        file_psimple.close()
        file_pstopwords = open((rep+"Methode Harrathi\pstopwords\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        num_file_pstopwords = sum(1 for line in file_pstopwords)
        file_pstopwords.close()
        file_pstopwords = open((rep+"Methode Harrathi\\pstopwords\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        pstopwordslist = []
        for j in range(0,num_file_pstopwords):
            line = file_pstopwords.readline().strip().split(" , ")
            line[1] = np.double(line[1])
            pstopwordslist.append(line)
        file_pstopwords.close()



        creerRepertoire(rep+"Methode Harrathi\\2gramm")
        file_2gramm = open((rep+"Methode Harrathi\\2gramm\%s.txt")%(all_files_names[i]), "w", encoding="utf8")

        bigram_measures = BigramAssocMeasures()
        finder2 = BigramCollocationFinder.from_words(termes[i])
        #finder.apply_word_filter(lambda x: x in stop_words)
        word_filter2 = lambda w1, w2: w1 in stop_words
        finder2.apply_ngram_filter(word_filter2)
        finder2.apply_freq_filter(1)
        #print('word1  , word2,  PMI')
        list2gramm = []
    
        for row in finder2.score_ngrams(bigram_measures.pmi):
            data = (*row[0],row[1])
            list2gramm.append(row[0])
            if len(data[0]) > 0 and  len(data[1]) > 0:
                if data[1] in stop_words:
                    datalist = list(data)
                    #print(datalist)
                    j = 0
                    while True:
                        pm1 = psimplelist[j][1]
                        j += 1
                        if datalist[0] == psimplelist[j-1][0]:
                            #print(psimplelist[j-1][0])
                            break
                    j = 0
                    while True:
                        pm2 = pstopwordslist[j][1]
                        j += 1
                        if datalist[1] == pstopwordslist[j-1][0]:
                            #print(pstopwordslist[j-1][0])
                            break
                    datalist[2] = _log2((2**(datalist[2]) * pm2) / pm1)
                    datalist = tuple(datalist)
                    file_2gramm.write(str(datalist))
                    file_2gramm.write("\n")
                else:
                    file_2gramm.write(str(data))
                    file_2gramm.write("\n")
        file_2gramm.close()

        creerRepertoire(rep + "Methode Harrathi\\3gramm")
        file_3gramm = open((rep+"Methode Harrathi\\3gramm\%s.txt")%(all_files_names[i]), "w", encoding="utf8")
    
        trigram_measures = TrigramAssocMeasures()
        finder3 = TrigramCollocationFinder.from_words(termes[i])
        #finder.apply_word_filter(lambda x: x in stop_words)
        word_filter3 = lambda w1, w2, w3: (w1, w2) not in list2gramm
        finder3.apply_ngram_filter(word_filter3)
        finder3.apply_freq_filter(2)
        #print('word1  , word2,  PMI')
        list3gramm = []

        for row in finder3.score_ngrams(trigram_measures.pmi):
            data = (*row[0],row[1])
            list3gramm.append(row[0])
            if len(data[0]) > 0 and  len(data[1]) > 0:
                if data[2] in stop_words:
                    datalist = list(data)
                    #print(datalist)
                    j = 0
                    while True:
                        pm2 = pstopwordslist[j][1]
                        j += 1
                        if datalist[2] == pstopwordslist[j-1][0]:
                            #print(pstopwordslist[j-1][0])
                            break
                    pm1 = cal_p2(datalist[0],datalist[1]) / num_words_corpus
                    datalist[3] = _log2((2**(datalist[3]) * pm2) / pm1)
                    datalist = tuple(datalist)
                    #print(datalist)
                    file_3gramm.write(str(datalist))
                    file_3gramm.write("\n")
                else:
                    #print(data)
                    file_3gramm.write(str(data))
                    file_3gramm.write("\n")
        file_3gramm.close()


        creerRepertoire(rep + "Methode Harrathi\\4gramm")
        file_4gramm = open((rep+"Methode Harrathi\\4gramm\%s.txt")%(all_files_names[i]), "w", encoding="utf8")
    
        quadgram_measures = QuadgramAssocMeasures()
        finder4 = QuadgramCollocationFinder.from_words(termes[i])
        #finder.apply_word_filter(lambda x: x in stop_words)
        word_filter4 = lambda w1, w2, w3, w4: (w1, w2, w3) not in list3gramm
        finder4.apply_ngram_filter(word_filter4)
        finder4.apply_freq_filter(2)
        #print('word1  , word2,  PMI')
        for row in finder4.score_ngrams(quadgram_measures.pmi):
            data = (*row[0],row[1])
            if len(data[0]) > 0 and  len(data[1]) > 0:
                if data[3] in stop_words:
                    datalist = list(data)
                    #print(datalist)
                    j = 0
                    while True:
                        pm2 = pstopwordslist[j][1]
                        j += 1
                        if datalist[3] == pstopwordslist[j-1][0]:
                            #print(pstopwordslist[j-1][0])
                            break
                    pm1 = cal_p3(datalist[0],datalist[1],datalist[2]) / num_words_corpus
                    datalist[4] = _log2((2**(datalist[4]) * pm2) / pm1)
                    datalist = tuple(datalist)
                    #print(datalist)
                    file_4gramm.write(str(datalist))
                    file_4gramm.write("\n")
                else:
                    #print(data)
                    file_4gramm.write(str(data))
                    file_4gramm.write("\n")
        file_4gramm.close()
        #print("fin")
        i = i+1
        print(f"boucle 4 ngrammes {i} fois")


    def takeLast(elem):
        return np.double(elem[-1])


    for i in range(N):

        len_doc = len(termes[i])

        file_simple = open((rep+"Methode Harrathi\\termes\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        num_file_simple = sum(1 for line in file_simple)
        file_simple.close()
        file_simple = open((rep+"Methode Harrathi\\termes\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        simplelist = []
        doclist = []
        for j in range(0,num_file_simple):
            line = file_simple.readline().strip().split(" , ")
            line[1] = np.double(line[1])
            doclist.append(line)
            simplelist.append(line[0])
        file_simple.close()
    
        file_psimple = open((rep+"Methode Harrathi\ptermes\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        num_file_psimple = sum(1 for line in file_psimple)
        file_psimple.close()
        file_psimple = open((rep+"Methode Harrathi\\ptermes\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        psimplelist = []
        for j in range(0,num_file_psimple):
            line = file_psimple.readline().strip().split(" , ")
            line[1] = np.double(line[1])
            psimplelist.append(line)
        file_psimple.close()

        file_2gramm = open((rep+"Methode Harrathi\\2gramm\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        num_file_2gramm = sum(1 for line in file_2gramm)
        file_2gramm.close()
        file_2gramm = open((rep+"Methode Harrathi\\2gramm\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        for j in range(0,num_file_2gramm):
            line = file_2gramm.readline().strip().replace("(","").replace(")","").replace("', '"," ").replace("'","").split(", ")
            n = nbr_doc(line[0])
            tf = tf_comp(line[0], i)
            p = cal_pond(N, n, tf, len_doc, moy_len_doc)
            l = lang_simple(line[0])
            som = som_simple(line[0])
            pond = pond_comp(l, p, som)
            line[1] = pond
            doclist.append(line)
        file_2gramm.close()

        file_3gramm = open((rep+"Methode Harrathi\\3gramm\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        num_file_3gramm = sum(1 for line in file_3gramm)
        file_3gramm.close()
        file_3gramm = open((rep+"Methode Harrathi\\3gramm\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        for j in range(0,num_file_3gramm):
            line = file_3gramm.readline().strip().replace("(","").replace(")","").replace("', '"," ").replace("'","").split(", ")
            n = nbr_doc(line[0])
            tf = tf_comp(line[0], i)
            p = cal_pond(N, n, tf, len_doc, moy_len_doc)
            l = lang_simple(line[0])
            som = som_simple(line[0])
            pond = pond_comp(l, p, som)
            line[1] = pond
            doclist.append(line)
        file_3gramm.close()

        file_4gramm = open((rep+"Methode Harrathi\\4gramm\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        num_file_4gramm = sum(1 for line in file_4gramm)
        file_4gramm.close()
        file_4gramm = open((rep+"Methode Harrathi\\4gramm\%s.txt")%(all_files_names[i]), "r", encoding="utf8")
        for j in range(0,num_file_4gramm):
            line = file_4gramm.readline().strip().replace("(","").replace(")","").replace("', '"," ").replace("'","").split(", ")
            n = nbr_doc(line[0])
            tf = tf_comp(line[0], i)
            p = cal_pond(N, n, tf, len_doc, moy_len_doc)
            l = lang_simple(line[0])
            som = som_simple(line[0])
            pond = pond_comp(l, p, som)
            line[1] = pond
            doclist.append(line)
        file_4gramm.close()


        doclist.sort(key=takeLast, reverse=True)

    
        Mcle = open((rep+"keys\%s.key")%(all_files_names[i]), 'r', encoding="utf8")
        num_Mcle = sum(1 for line in Mcle)    #70

        Mcle.close()

        creerRepertoire(rep + "Methode Harrathi\\keys")
        file_keys = open((rep+"Methode Harrathi\\keys\%s.txt")%(all_files_names[i]), "w", encoding="utf8")
    
        for j in range(num_Mcle):
            file_keys.write(doclist[j][0])
            if j != num_Mcle-1:
                file_keys.write("\n")
        file_keys.close

        print(f"boucle ecriture {i+1} fois")


###########################################################
def vider_repertoire(repertoire):
    # Vérifier si le répertoire existe
    if os.path.exists(repertoire):
        # Obtenir la liste des fichiers dans le répertoire
        fichiers = os.listdir(repertoire)

        # Parcourir la liste des fichiers et les supprimer un par un
        for fichier in fichiers:
            chemin_fichier = os.path.join(repertoire, fichier)
            os.remove(chemin_fichier)
        
        print("Répertoire vidé avec succès.")
    else:
        #repertoire_rake = "RAKE"
        if not os.path.exists(repertoire):
            os.makedirs(repertoire)

        print("Le répertoire spécifié a été créer.")

def ouvrir_fichiers_Rake(repertoire):

    rep_doc = repertoire + "/docsutf8"
    # Vérifier si un répertoire a été sélectionné
    if rep_doc:
        # Récupérer la liste des fichiers texte du répertoire
        filenames = [os.path.join(rep_doc, f) for f in os.listdir(rep_doc) if os.path.isfile(os.path.join(rep_doc, f)) and f.endswith(".txt")]
        
        # Vérifier si des fichiers ont été trouvés dans le répertoire
        if filenames:
            contenus = []
            
            # Ouvrir les fichiers et lire leur contenu
            for filename in filenames:
                with open(filename, "r", encoding="utf8") as file:
                #with open(filename, "r") as file:
                    #contenu = nettoyer(file.read())
                    contenu = file.read()
                    #print(contenu)
                    #contenu = nettoyer(contenu)
                    #print(contenu)
                    contenus.append(contenu)
        
            liste_termes = extraire_termes_rake(contenus)
            #print(liste_termes)
            # Chemin du répertoire à vider
            repertoire_rake = repertoire+"/RAKE"
            vider_repertoire(repertoire_rake)

            # Écrire les listes de termes dans des fichiers individuels
            for i, termes in enumerate(liste_termes):
                nom_fichier = os.path.join(repertoire_rake, f"{os.path.splitext(os.path.basename(filenames[i]))[0]}.txt")
                with open(nom_fichier, "w", encoding="utf8") as file:
                    for terme in termes:
                        file.write(f"{terme}\n")  

            print("Les termes ont été stockés dans des fichiers individuels dans le répertoire 'RAKE'.")

        else:
            # Afficher un message d'erreur si aucun fichier n'a été trouvé dans le répertoire
            messagebox.showerror("Erreur", "Aucun fichier texte trouvé dans le répertoire sélectionné.")
    
    else:
        # Afficher un message d'erreur si aucun répertoire n'a été sélectionné
        messagebox.showerror("Erreur", "Aucun répertoire sélectionné.")

def ouvrir_fichiers_Yake(repertoire):
    #repertoire = "docsutf8"  # Spécifier le chemin absolu du répertoire contenant les fichiers
    rep_doc = repertoire + "/docsutf8"
    # Récupérer la liste des fichiers texte du répertoire
    filenames = [os.path.join(rep_doc, f) for f in os.listdir(rep_doc) if os.path.isfile(os.path.join(rep_doc, f)) and f.endswith(".txt")]
    
    # Vérifier si des fichiers ont été trouvés dans le répertoire
    if filenames:
        contenus = []
        
        # Ouvrir les fichiers et lire leur contenu
        for filename in filenames:
            #with open(filename, "r", encoding="utf8") as file:
            with open(filename, "r") as file:
                contenu = file.read()
                contenus.append(contenu)
        
        liste_termes = extraire_termes_yake(contenus)

        # Chemin du répertoire à vider
        repertoire_yake = repertoire+"/YAKE"
        vider_repertoire(repertoire_yake)

        # Écrire les listes de termes dans des fichiers individuels
        for i, termes in enumerate(liste_termes):
            nom_fichier = os.path.join(repertoire_yake, f"{os.path.splitext(os.path.basename(filenames[i]))[0]}.txt")
            with open(nom_fichier, "w", encoding="utf8") as file:
                for terme in termes:
                    file.write(f"{terme}\n")  

        print("Les termes ont été stockés dans des fichiers individuels dans le répertoire 'YAKE'.")

    else:
        # Afficher un message d'erreur si aucun fichier n'a été trouvé dans le répertoire
        messagebox.showerror("Erreur", "Aucun fichier texte trouvé dans le répertoire spécifié.")

def ouvrir_fichiers_TfIdf(repertoire):
    #repertoire = "docsutf8"  # Spécifier le chemin absolu du répertoire contenant les fichiers
    
    rep_doc = repertoire + "/docsutf8"

    # Récupérer la liste des fichiers texte du répertoire
    filenames = [os.path.join(rep_doc, f) for f in os.listdir(rep_doc) if os.path.isfile(os.path.join(rep_doc, f)) and f.endswith(".txt")]
    
    # Vérifier si des fichiers ont été trouvés dans le répertoire
    if filenames:
        contenus = []
        
        # Ouvrir les fichiers et lire leur contenu
        for filename in filenames:
            #with open(filename, "r", encoding="utf8") as file:
            with open(filename, "r") as file:
                contenu = file.read()
                contenu = nettoyer(contenu)
                contenus.append(contenu)
        
        liste_termes = extraire_termes_TfIdf(contenus)

        repertoire_tfidf = repertoire + "/TFIDF"  # Chemin du répertoire à vider
        vider_repertoire(repertoire_tfidf)

        # Écrire les listes de termes dans des fichiers individuels
        for i, termes in enumerate(liste_termes):
            nom_fichier = os.path.join(repertoire_tfidf, f"{os.path.splitext(os.path.basename(filenames[i]))[0]}.txt")
            with open(nom_fichier, "w", encoding="utf8") as file:
                for terme in termes:
                    file.write(f"{terme}\n")  

        print("Les termes ont été stockés dans des fichiers individuels dans le répertoire 'TFIDF'.")

    else:
        # Afficher un message d'erreur si aucun fichier n'a été trouvé dans le répertoire
        messagebox.showerror("Erreur", "Aucun fichier texte trouvé dans le répertoire spécifié.")

def ouvrir_fichiers_Methode(repertoire):
    #repertoire = "docsutf8"  # Spécifier le chemin absolu du répertoire contenant les fichiers
    rep_doc = repertoire + "/docsutf8"
    # Récupérer la liste des fichiers texte du répertoire
    filenames = [os.path.join(rep_doc, f) for f in os.listdir(rep_doc) if os.path.isfile(os.path.join(rep_doc, f)) and f.endswith(".txt")]
    
    # Vérifier si des fichiers ont été trouvés dans le répertoire
    if filenames:
        contenus = []
        nlp = spacy.load('en_core_web_sm') 
        
        # Ouvrir les fichiers et lire leur contenu
        for filename in filenames:
            #with open(filename, "r", encoding="utf8") as file:
            with open(filename, "r") as file:
                contenu = file.read()
                contenus.append(contenu)
        
        liste_termes = extraction_methode(contenus, nlp) 
        print("termes extraits")
        repertoire_methode = repertoire + "/METHODE PROPOSEE"  # Chemin du répertoire à vider
        vider_repertoire(repertoire_methode) 

        # Écrire les listes de termes dans des fichiers individuels
        for i, termes in enumerate(liste_termes):
            nom_fichier = os.path.join(repertoire_methode, f"{os.path.splitext(os.path.basename(filenames[i]))[0]}.txt")
            with open(nom_fichier, "w", encoding="utf8") as file:
                for terme in termes:
                    file.write(f"{terme}\n")  
       
        print("Les termes ont été stockés dans des fichiers individuels dans le répertoire 'METHODE PROPOSEE'.")

    else:
        # Afficher un message d'erreur si aucun fichier n'a été trouvé dans le répertoire
        messagebox.showerror("Erreur", "Aucun fichier texte trouvé dans le répertoire spécifié.")

def comparer_fichiers_manuel(repertoire,rep):
    repertoire_manuel = rep + "/MANUEL"  # Chemin du répertoire "MANUEL"
    repertoire_rake = repertoire  # Chemin du répertoire 
    
    # Récupérer la liste des fichiers texte du répertoire "MANUEL"
    filenames_manuel = [f for f in os.listdir(repertoire_manuel) if os.path.isfile(os.path.join(repertoire_manuel, f)) and f.endswith(".txt")]
    liste_r = []
    liste_p = []
    liste_m = []
    #print(filenames_manuel)
    for filename in filenames_manuel:
        #print(filename)
        entier = 0
        chemin_manuel = os.path.join(repertoire_manuel, filename)
        chemin_rake = os.path.join(repertoire_rake, filename)
        
        if(rep != "CORPUS02"):
            if os.path.isfile(chemin_rake):
                with open(chemin_manuel, "r",encoding="utf8") as fichier_manuel, open(chemin_rake, "r",encoding="utf8") as fichier_rake:
                    lignes_manuel = fichier_manuel.readlines()
                    lignes_rake = fichier_rake.readlines()[:len(lignes_manuel)]
                
                    for ligne_manuel in lignes_manuel:
                        ligne_manuel = ligne_manuel.lower() 

                        if any(ligne_manuel in ligne_rake.lower() for ligne_rake in lignes_rake):
                            entier = entier +1 
  
                    r = entier / len(lignes_manuel)
                    p = entier / len(lignes_rake)
            
                    if r == 0 and p == 0:
                        m = 0
                    else:
                        m = (2*r*p) / (r+p)

                    liste_r.append(r)
                    liste_p.append(p)
                    liste_m.append(m)
            else:
                print("n'existe pas")
        else :
            if os.path.isfile(chemin_rake):
                with open(chemin_manuel, "r") as fichier_manuel, open(chemin_rake, "r") as fichier_rake:
                    lignes_manuel = fichier_manuel.readlines()
                    lignes_rake = fichier_rake.readlines()[:len(lignes_manuel)]
                
                    for ligne_manuel in lignes_manuel:
                        ligne_manuel = ligne_manuel.lower() 

                        if any(ligne_manuel in ligne_rake.lower() for ligne_rake in lignes_rake):
                            entier = entier +1 
  
                    r = entier / len(lignes_manuel)
                    p = entier / len(lignes_rake)
            
                    if r == 0 and p == 0:
                        m = 0
                    else:
                        m = (2*r*p) / (r+p)

                    liste_r.append(r)
                    liste_p.append(p)
                    liste_m.append(m)
            else:
                print("n'existe pas")

    if len(liste_r) != 0 : 
        moyenner = sum(liste_r) / len(liste_r)
    else:
        moyenner = 0

    if len(liste_p) !=0 :
        moyennep = sum(liste_p) / len(liste_p)
    else:
        moyennep = 0

    if len(liste_m) !=0 :
        moyennem = sum(liste_m) / len(liste_m)
    else:
        moyennem = 0

    tableau = [moyenner, moyennep, moyennem]
    return tableau

def calculer(repertoire):
    
    tableau_total = []

    mon_tableau_rake = comparer_fichiers_manuel(repertoire + "/RAKE", repertoire)
    tableau_total.extend(mon_tableau_rake)

    mon_tableau_yake = comparer_fichiers_manuel(repertoire + "/YAKE", repertoire)
    tableau_total.extend(mon_tableau_yake)

    mon_tableau_tfidf = comparer_fichiers_manuel(repertoire + "/TFIDF", repertoire)
    tableau_total.extend(mon_tableau_tfidf)

    mon_tableau_methode = comparer_fichiers_manuel(repertoire + "/METHODE", repertoire)
    tableau_total.extend(mon_tableau_methode)

    mon_tableau_nmethode = comparer_fichiers_manuel(repertoire + "/Inspec/terms/newmethod/keys", repertoire)
    tableau_total.extend(mon_tableau_nmethode)

    return tableau_total

def moyenne_tableaux(*tableaux):

    # Vérifier si un seul tableau est fourni 
    if len(tableaux) == 1:
        return tableaux[0]
    
    # Vérifier si au moins deux tableaux sont fournis
    if len(tableaux) < 2:
        raise ValueError("Veuillez fournir au moins deux tableaux pour calculer la moyenne.")

    # Vérifier si tous les tableaux ont la même longueur
    longueur = len(tableaux[0])
    if not all(len(t) == longueur for t in tableaux):
        raise ValueError("Les tableaux doivent avoir la même longueur.")

    # Calculer la moyenne élément par élément
    moyenne = []
    for i in range(longueur):
        somme_element = sum(t[i] for t in tableaux)
        moyenne_element = somme_element / len(tableaux)
        moyenne.append(moyenne_element)

    return moyenne

def comparer_fichiers_man(repertoire,nombre,rep):
    repertoire_manuel = rep + "/MANUEL"  # Chemin du répertoire "MANUEL"
    repertoire_rake = repertoire  # Chemin du répertoire 
    
    # Récupérer la liste des fichiers texte du répertoire "MANUEL"
    filenames_manuel = [f for f in os.listdir(repertoire_manuel) if os.path.isfile(os.path.join(repertoire_manuel, f)) and f.endswith(".txt")]
    liste_r = []
    liste_p = []
    liste_m = []
    #print(filenames_manuel)
    for filename in filenames_manuel:
        #print(filename)
        entier = 0
        chemin_manuel = os.path.join(repertoire_manuel, filename)
        chemin_rake = os.path.join(repertoire_rake, filename)
        
        if(rep != "CORPUS02"):
            if os.path.isfile(chemin_rake):
                with open(chemin_manuel, "r", encoding="utf8") as fichier_manuel, open(chemin_rake, "r", encoding="utf8") as fichier_rake:
                    lignes_manuel = fichier_manuel.readlines()
                    lignes_rake = fichier_rake.readlines()[:nombre]
                
                    for ligne_manuel in lignes_manuel:
                        ligne_manuel = ligne_manuel.lower()  # Supprimer les espaces et sauts de ligne

                        if any(ligne_manuel in ligne_rake.lower() for ligne_rake in lignes_rake):
                            entier = entier +1 

                    r = entier / len(lignes_manuel)
                    p = entier / nombre
            
                    if r == 0 and p == 0:
                        m = 0
                    else:
                        m = (2*r*p) / (r+p)

                    liste_r.append(r)
                    liste_p.append(p)
                    liste_m.append(m)

        else:
            if os.path.isfile(chemin_rake):
                with open(chemin_manuel, "r") as fichier_manuel, open(chemin_rake, "r") as fichier_rake:
                    lignes_manuel = fichier_manuel.readlines()
                    lignes_rake = fichier_rake.readlines()[:nombre]
                
                    for ligne_manuel in lignes_manuel:
                        ligne_manuel = ligne_manuel.lower()  # Supprimer les espaces et sauts de ligne

                        if any(ligne_manuel in ligne_rake.lower() for ligne_rake in lignes_rake):
                            entier = entier +1 

                    r = entier / len(lignes_manuel)
                    p = entier / nombre
            
                    if r == 0 and p == 0:
                        m = 0
                    else:
                        m = (2*r*p) / (r+p)

                    liste_r.append(r)
                    liste_p.append(p)
                    liste_m.append(m)



    if len(liste_r) != 0 : 
        moyenner = sum(liste_r) / len(liste_r)
    else:
        moyenner = 0

    if len(liste_p) !=0 :
        moyennep = sum(liste_p) / len(liste_p)
    else:
        moyennep = 0

    if len(liste_m) !=0 :
        moyennem = sum(liste_m) / len(liste_m)
    else:
        moyennem = 0

    tableau = [moyenner, moyennep, moyennem]
    return tableau

def ouvrir_corpus_rake():

    # Demander à l'utilisateur de sélectionner un dossier
    selected_folder = askdirectory()

    # Extraire le nom du dossier à partir du chemin complet
    folder_name = selected_folder.split("/")[-1]

    ouvrir_fichiers_Rake(folder_name)

def ouvrir_corpus_yake():

    # Demander à l'utilisateur de sélectionner un dossier
    selected_folder = askdirectory()

    # Extraire le nom du dossier à partir du chemin complet
    folder_name = selected_folder.split("/")[-1]
    
    ouvrir_fichiers_Yake(folder_name)

def ouvrir_corpus_TfIdf():

    # Demander à l'utilisateur de sélectionner un dossier
    selected_folder = askdirectory()

    # Extraire le nom du dossier à partir du chemin complet
    folder_name = selected_folder.split("/")[-1]
    
    ouvrir_fichiers_TfIdf(folder_name)

def ouvrir_corpus_Methode():

    # Demander à l'utilisateur de sélectionner un dossier
    selected_folder = askdirectory()

    # Extraire le nom du dossier à partir du chemin complet
    folder_name = selected_folder.split("/")[-1]

    ouvrir_fichiers_Methode(folder_name)

def ouvrir_corpus_nmethode():

    # Demander à l'utilisateur de sélectionner un dossier
    selected_folder = askdirectory()

    # Extraire le nom du dossier à partir du chemin complet
    folder_name = selected_folder.split("/")[-1]

    #extraction_nmethode(folder_name+"/Inspec/",folder_name+"\\")
    extraction_nmethode(folder_name+ "/", folder_name + "\\")


def ouvrir_corpus_calcul():

    # Demander à l'utilisateur de sélectionner un dossier
    selected_folder = askdirectory()

    # Extraire le nom du dossier à partir du chemin complet
    folder_name = selected_folder.split("/")[-1]

    tableau_total1 = calculer(folder_name)

    resultat = moyenne_tableaux(tableau_total1)
    
    print(resultat)

    # Créer une fenêtre
    fenetre = Tk()
    fenetre.title('Précision - Rappel - FMesure')
    # Créer un tableau avec trois colonnes
    tableau = ttk.Treeview(fenetre, columns=('Colonne 1', 'Colonne 2', 'Colonne 3'))
    # Définir les en-têtes de colonnes
    tableau.heading('#0', text='Index')
    tableau.heading('Colonne 1', text='Rappel')
    tableau.heading('Colonne 2', text='Precision')
    tableau.heading('Colonne 3', text='F-mesure')

    # Ajouter des données au tableau
    tableau.insert(parent='', index='end', iid=0, text='RAKE', values=(resultat[0], resultat[1], resultat[2]))
    tableau.insert(parent='', index='end', iid=1, text='YAKE', values=(resultat[3], resultat[4], resultat[5]))
    tableau.insert(parent='', index='end', iid=2, text='TF-IDF', values=(resultat[6], resultat[7], resultat[8]))
    tableau.insert(parent='', index='end', iid=3, text='METHODE ADOPTE', values=(resultat[9], resultat[10], resultat[11]))
    tableau.insert(parent='', index='end', iid=4, text='Méthode de Harrath ', values=(resultat[12], resultat[13], resultat[14]))

    # Afficher le tableau
    tableau.pack()

    fenetre.mainloop()
    
def graphe (repm,repertoire):

    tableau_total = []

    mon_tableau_rake_30 = comparer_fichiers_man(repm,30,repertoire)

    tableau_total.extend(mon_tableau_rake_30)

    mon_tableau_rake_40 = comparer_fichiers_man(repm,40,repertoire)

    tableau_total.extend(mon_tableau_rake_40)

    mon_tableau_rake_50 = comparer_fichiers_man(repm,50,repertoire)

    tableau_total.extend(mon_tableau_rake_50)

    mon_tableau_rake_60 = comparer_fichiers_man(repm,60,repertoire)

    tableau_total.extend(mon_tableau_rake_60)

    mon_tableau_rake_70 = comparer_fichiers_man(repm,70,repertoire)

    tableau_total.extend(mon_tableau_rake_70)

    return tableau_total

def graphe_corpus():

    # Demander à l'utilisateur de sélectionner un dossier
    selected_folder = askdirectory()

    # Extraire le nom du dossier à partir du chemin complet
    folder_name = selected_folder.split("/")[-1]

    import tkinter as tk
    
    #rake
    main_window = Tk()
    main_window.title('Mesure RAKE a partiir d\'un corpus')
    tableau01 = graphe(folder_name+"/RAKE",folder_name)
    tab = moyenne_tableaux(tableau01)
    
    data ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Rappel':[tab[0],tab[3],tab[6],tab[9],tab[12]],
    }

    dataa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Presition':[tab[1],tab[4],tab[7],tab[10],tab[13]],
    }

    dataaa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Fmessure':[tab[2],tab[5],tab[8],tab[11],tab[14]],
    }

    dataframe = pd.DataFrame(data)
    dataframe1 = pd.DataFrame(dataa)
    dataframe2 = pd.DataFrame(dataaa)

    figure = plt.Figure(figsize=(5,4),dpi=100)
    figure_plot = figure.add_subplot(1,1,1)
    figure_plot.set_ylabel('score')
    line_graph = FigureCanvasTkAgg(figure,main_window)
    line_graph.get_tk_widget().pack(side=tk.LEFT,fill=tk.BOTH)

    dataframe = dataframe[['Nombre de mots extrait','Rappel']].groupby('Nombre de mots extrait').sum()
    dataframe.plot(kind='line',legend=True,ax=figure_plot,color='r',marker='o',fontsize=10)

    dataframe1 = dataframe1[['Nombre de mots extrait','Presition']].groupby('Nombre de mots extrait').sum()
    dataframe1.plot(kind='line',legend=True,ax=figure_plot,color='blue',marker='o',fontsize=10)    

    dataframe2 = dataframe2[['Nombre de mots extrait','Fmessure']].groupby('Nombre de mots extrait').sum()
    dataframe2.plot(kind='line',legend=True,ax=figure_plot,color='green',marker='o',fontsize=10)    

    figure_plot.set_title('RAKE')

    #YAKE

    main_window1 = Tk()
    main_window1.title('Mesure YAKE a partiir d\'un corpus')
    tableau001 = graphe(folder_name+"/YAKE",folder_name)

    taby = moyenne_tableaux(tableau001)
    
    data ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Rappel':[taby[0],taby[3],taby[6],taby[9],taby[12]],
    }

    dataa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Presition':[taby[1],taby[4],taby[7],taby[10],taby[13]],
    }

    dataaa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Fmessure':[taby[2],taby[5],taby[8],taby[11],taby[14]],
    }

    dataframe = pd.DataFrame(data)
    dataframe1 = pd.DataFrame(dataa)
    dataframe2 = pd.DataFrame(dataaa)

    figure = plt.Figure(figsize=(5,4),dpi=100)
    figure_plot = figure.add_subplot(1,1,1)
    figure_plot.set_ylabel('Unemployment Rate')
    line_graph = FigureCanvasTkAgg(figure,main_window1)
    line_graph.get_tk_widget().pack(side=tk.LEFT,fill=tk.BOTH)

    dataframe = dataframe[['Nombre de mots extrait','Rappel']].groupby('Nombre de mots extrait').sum()
    dataframe.plot(kind='line',legend=True,ax=figure_plot,color='r',marker='o',fontsize=10)

    dataframe1 = dataframe1[['Nombre de mots extrait','Presition']].groupby('Nombre de mots extrait').sum()
    dataframe1.plot(kind='line',legend=True,ax=figure_plot,color='blue',marker='o',fontsize=10)    

    dataframe2 = dataframe2[['Nombre de mots extrait','Fmessure']].groupby('Nombre de mots extrait').sum()
    dataframe2.plot(kind='line',legend=True,ax=figure_plot,color='green',marker='o',fontsize=10)    

    figure_plot.set_title('YAKE')

    #TFIDF

    main_window2 = Tk()
    main_window2.title('Mesure TFIDF a partiir d\'un corpus')
    tableauTF01 = graphe(folder_name+"/TFIDF",folder_name)

    tabTF = moyenne_tableaux(tableauTF01)
    
    data ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Rappel':[tabTF[0],tabTF[3],tabTF[6],tabTF[9],tabTF[12]],
    }

    dataa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Presition':[tabTF[1],tabTF[4],tabTF[7],tabTF[10],tabTF[13]],
    }

    dataaa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Fmessure':[tabTF[2],tabTF[5],tabTF[8],tabTF[11],tabTF[14]],
    }

    dataframe = pd.DataFrame(data)
    dataframe1 = pd.DataFrame(dataa)
    dataframe2 = pd.DataFrame(dataaa)

    figure = plt.Figure(figsize=(5,4),dpi=100)
    figure_plot = figure.add_subplot(1,1,1)
    figure_plot.set_ylabel('Unemployment Rate')
    line_graph = FigureCanvasTkAgg(figure,main_window2)
    line_graph.get_tk_widget().pack(side=tk.LEFT,fill=tk.BOTH)

    dataframe = dataframe[['Nombre de mots extrait','Rappel']].groupby('Nombre de mots extrait').sum()
    dataframe.plot(kind='line',legend=True,ax=figure_plot,color='r',marker='o',fontsize=10)

    dataframe1 = dataframe1[['Nombre de mots extrait','Presition']].groupby('Nombre de mots extrait').sum()
    dataframe1.plot(kind='line',legend=True,ax=figure_plot,color='blue',marker='o',fontsize=10)    

    dataframe2 = dataframe2[['Nombre de mots extrait','Fmessure']].groupby('Nombre de mots extrait').sum()
    dataframe2.plot(kind='line',legend=True,ax=figure_plot,color='green',marker='o',fontsize=10)    

    figure_plot.set_title('TFIDF')

    #METHODE

    main_window3 = Tk()
    main_window3.title('Mesure METHODE a partir d\'un corpus')
    tableauM1 = graphe(folder_name+"/METHODE",folder_name)

    tabM = moyenne_tableaux(tableauM1)
    
    data ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Rappel':[tabM[0],tabM[3],tabM[6],tabM[9],tabM[12]],
    }

    dataa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Presition':[tabM[1],tabM[4],tabM[7],tabM[10],tabM[13]],
    }

    dataaa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Fmessure':[tabM[2],tabM[5],tabM[8],tabM[11],tabM[14]],
    }

    dataframe = pd.DataFrame(data)
    dataframe1 = pd.DataFrame(dataa)
    dataframe2 = pd.DataFrame(dataaa)

    figure = plt.Figure(figsize=(5,4),dpi=100)
    figure_plot = figure.add_subplot(1,1,1)
    figure_plot.set_ylabel('Unemployment Rate')
    line_graph = FigureCanvasTkAgg(figure,main_window3)
    line_graph.get_tk_widget().pack(side=tk.LEFT,fill=tk.BOTH)

    dataframe = dataframe[['Nombre de mots extrait','Rappel']].groupby('Nombre de mots extrait').sum()
    dataframe.plot(kind='line',legend=True,ax=figure_plot,color='r',marker='o',fontsize=10)

    dataframe1 = dataframe1[['Nombre de mots extrait','Presition']].groupby('Nombre de mots extrait').sum()
    dataframe1.plot(kind='line',legend=True,ax=figure_plot,color='blue',marker='o',fontsize=10)    

    dataframe2 = dataframe2[['Nombre de mots extrait','Fmessure']].groupby('Nombre de mots extrait').sum()
    dataframe2.plot(kind='line',legend=True,ax=figure_plot,color='green',marker='o',fontsize=10)    

    figure_plot.set_title('METHODE')

    #NMETHODE

    main_window4 = Tk()
    main_window4.title('Mesure NMETHODE a partir d\'un corpus')

    tableauM1 = graphe(folder_name+"/Inspec/terms/newmethod/keys",folder_name)

    tabN = moyenne_tableaux(tableauM1)
    
    data ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Rappel':[tabN[0],tabN[3],tabN[6],tabN[9],tabN[12]],
    }

    dataa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Presition':[tabN[1],tabN[4],tabN[7],tabN[10],tabN[13]],
    }

    dataaa ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Fmessure':[tabN[2],tabN[5],tabN[8],tabN[11],tabN[14]],
    }

    dataframe = pd.DataFrame(data)
    dataframe1 = pd.DataFrame(dataa)
    dataframe2 = pd.DataFrame(dataaa)

    figure = plt.Figure(figsize=(5,4),dpi=100)
    figure_plot = figure.add_subplot(1,1,1)
    figure_plot.set_ylabel('Unemployment Rate')
    line_graph = FigureCanvasTkAgg(figure,main_window4)
    line_graph.get_tk_widget().pack(side=tk.LEFT,fill=tk.BOTH)

    dataframe = dataframe[['Nombre de mots extrait','Rappel']].groupby('Nombre de mots extrait').sum()
    dataframe.plot(kind='line',legend=True,ax=figure_plot,color='r',marker='o',fontsize=10)

    dataframe1 = dataframe1[['Nombre de mots extrait','Presition']].groupby('Nombre de mots extrait').sum()
    dataframe1.plot(kind='line',legend=True,ax=figure_plot,color='blue',marker='o',fontsize=10)    

    dataframe2 = dataframe2[['Nombre de mots extrait','Fmessure']].groupby('Nombre de mots extrait').sum()
    dataframe2.plot(kind='line',legend=True,ax=figure_plot,color='green',marker='o',fontsize=10)    

    figure_plot.set_title('Méthode de Harrath')

    #rappel

    main_window5 = Tk()
    main_window5.title('Mesure Rappel a partir d\'un corpus')
    data1 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'RAKE':[tab[0],tab[3],tab[6],tab[9],tab[12]],
    }
    data2 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'YAKE':[taby[0],taby[3],taby[6],taby[9],taby[12]],
    }
    data3 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'TFIDF':[tabTF[0],tabTF[3],tabTF[6],tabTF[9],tabTF[12]],
    }
    data4 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'METHODE':[tabM[0],tabM[3],tabM[6],tabM[9],tabM[12]], 
    }
    data5 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Méthode de Harrath':[tabN[0],tabN[3],tabN[6],tabN[9],tabN[12]],
    }

    dataframe = pd.DataFrame(data1)
    dataframe1 = pd.DataFrame(data2)
    dataframe2 = pd.DataFrame(data3)
    dataframe3 = pd.DataFrame(data4)
    dataframe4 = pd.DataFrame(data5)

    figure = plt.Figure(figsize=(5,4),dpi=100)
    figure_plot = figure.add_subplot(1,1,1)
    figure_plot.set_ylabel('Unemployment Rate')
    line_graph = FigureCanvasTkAgg(figure,main_window5)
    line_graph.get_tk_widget().pack(side=tk.LEFT,fill=tk.BOTH)

    dataframe = dataframe[['Nombre de mots extrait','RAKE']].groupby('Nombre de mots extrait').sum()
    dataframe.plot(kind='line',legend=True,ax=figure_plot,color='r',marker='o',fontsize=10)

    dataframe1 = dataframe1[['Nombre de mots extrait','YAKE']].groupby('Nombre de mots extrait').sum()
    dataframe1.plot(kind='line',legend=True,ax=figure_plot,color='blue',marker='o',fontsize=10)    

    dataframe2 = dataframe2[['Nombre de mots extrait','TFIDF']].groupby('Nombre de mots extrait').sum()
    dataframe2.plot(kind='line',legend=True,ax=figure_plot,color='green',marker='o',fontsize=10)    

    dataframe3 = dataframe3[['Nombre de mots extrait','METHODE']].groupby('Nombre de mots extrait').sum()
    dataframe3.plot(kind='line',legend=True,ax=figure_plot,color='yellow',marker='o',fontsize=10)    

    dataframe4 = dataframe4[['Nombre de mots extrait','Méthode de Harrath']].groupby('Nombre de mots extrait').sum()
    dataframe4.plot(kind='line',legend=True,ax=figure_plot,color='black',marker='o',fontsize=10) 

    figure_plot.set_title('Rappel')

    #presition

    main_window6 = Tk()
    main_window6.title('Mesure Precision a partir d\'un corpus')

    data1 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'RAKE':[tab[1],tab[4],tab[7],tab[10],tab[13]],
    }

    data2 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'YAKE':[taby[1],taby[4],taby[7],taby[10],taby[13]],
    }

    data3 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'TFIDF':[tabTF[1],tabTF[4],tabTF[7],tabTF[10],tabTF[13]],
    }

    data4 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'METHODE':[tabM[1],tabM[4],tabM[7],tabM[10],tabM[13]],
    }

    data5 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Méthode de Harrath':[tabN[1],tabN[4],tabN[7],tabN[10],tabN[13]],
    }

    dataframe = pd.DataFrame(data1)
    dataframe1 = pd.DataFrame(data2)
    dataframe2 = pd.DataFrame(data3)
    dataframe3 = pd.DataFrame(data4)
    dataframe4 = pd.DataFrame(data5)

    figure = plt.Figure(figsize=(5,4),dpi=100)
    figure_plot = figure.add_subplot(1,1,1)
    figure_plot.set_ylabel('Unemployment Rate')
    line_graph = FigureCanvasTkAgg(figure,main_window6)
    line_graph.get_tk_widget().pack(side=tk.LEFT,fill=tk.BOTH)

    dataframe = dataframe[['Nombre de mots extrait','RAKE']].groupby('Nombre de mots extrait').sum()
    dataframe.plot(kind='line',legend=True,ax=figure_plot,color='r',marker='o',fontsize=10)

    dataframe1 = dataframe1[['Nombre de mots extrait','YAKE']].groupby('Nombre de mots extrait').sum()
    dataframe1.plot(kind='line',legend=True,ax=figure_plot,color='blue',marker='o',fontsize=10)    

    dataframe2 = dataframe2[['Nombre de mots extrait','TFIDF']].groupby('Nombre de mots extrait').sum()
    dataframe2.plot(kind='line',legend=True,ax=figure_plot,color='green',marker='o',fontsize=10)    

    dataframe3 = dataframe3[['Nombre de mots extrait','METHODE']].groupby('Nombre de mots extrait').sum()
    dataframe3.plot(kind='line',legend=True,ax=figure_plot,color='yellow',marker='o',fontsize=10)    

    dataframe4 = dataframe4[['Nombre de mots extrait','Méthode de Harrath']].groupby('Nombre de mots extrait').sum()
    dataframe4.plot(kind='line',legend=True,ax=figure_plot,color='black',marker='o',fontsize=10) 

    figure_plot.set_title('Presition')

    #Fmessure
    main_window7 = Tk()
    main_window7.title('Mesure Fmesure a partir d\'un corpus')

    data1 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'RAKE':[tab[2],tab[5],tab[8],tab[11],tab[14]],
    }

    data2 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'YAKE':[taby[2],taby[5],taby[8],taby[11],taby[14]],
    }

    data3 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'TFIDF':[tabTF[2],tabTF[5],tabTF[8],tabTF[11],tabTF[14]],
    }

    data4 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'METHODE':[tabM[2],tabM[5],tabM[8],tabM[11],tabM[14]],
    }

    data5 ={
        'Nombre de mots extrait':[30,40,50,60,70],
        'Méthode de Harrath':[tabN[2],tabN[5],tabN[8],tabN[11],tabN[14]],
    }

    dataframe = pd.DataFrame(data1)
    dataframe1 = pd.DataFrame(data2)
    dataframe2 = pd.DataFrame(data3)
    dataframe3 = pd.DataFrame(data4)
    dataframe4 = pd.DataFrame(data5)

    figure = plt.Figure(figsize=(5,4),dpi=100)
    figure_plot = figure.add_subplot(1,1,1)
    figure_plot.set_ylabel('Unemployment Rate')
    line_graph = FigureCanvasTkAgg(figure,main_window7)
    line_graph.get_tk_widget().pack(side=tk.LEFT,fill=tk.BOTH)

    dataframe = dataframe[['Nombre de mots extrait','RAKE']].groupby('Nombre de mots extrait').sum()
    dataframe.plot(kind='line',legend=True,ax=figure_plot,color='r',marker='o',fontsize=10)

    dataframe1 = dataframe1[['Nombre de mots extrait','YAKE']].groupby('Nombre de mots extrait').sum()
    dataframe1.plot(kind='line',legend=True,ax=figure_plot,color='blue',marker='o',fontsize=10)    

    dataframe2 = dataframe2[['Nombre de mots extrait','TFIDF']].groupby('Nombre de mots extrait').sum()
    dataframe2.plot(kind='line',legend=True,ax=figure_plot,color='green',marker='o',fontsize=10)    

    dataframe3 = dataframe3[['Nombre de mots extrait','METHODE']].groupby('Nombre de mots extrait').sum()
    dataframe3.plot(kind='line',legend=True,ax=figure_plot,color='yellow',marker='o',fontsize=10)    

    dataframe4 = dataframe4[['Nombre de mots extrait','Méthode de Harrath']].groupby('Nombre de mots extrait').sum()
    dataframe4.plot(kind='line',legend=True,ax=figure_plot,color='black',marker='o',fontsize=10) 

    figure_plot.set_title('FMESURE')

    main_window.mainloop()
    main_window1.mainloop()
    main_window2.mainloop()
    main_window3.mainloop()
    main_window4.mainloop()
    main_window5.mainloop()
    main_window6.mainloop()
    main_window7.mainloop()

def retour():
    fenetre.destroy()
    
# Créer un bouton pour ouvrir les deux fichiers texte et afficher leur contenu dans deux widgets Text différents
bouton = Button(fenetre, text="RAKE", command=ouvrir_corpus_rake, width=30, height=2)
bouton.place(x=100,y=100)

bouton1 = Button(fenetre, text="YAKE", command=ouvrir_corpus_yake, width=30, height=2)
bouton1.place(x=500,y=100)

bouton2 = Button(fenetre, text="TF-IDF", command=ouvrir_corpus_TfIdf, width=30, height=2)
bouton2.place(x=1000,y=100)

bouton3 = Button(fenetre, text="Methode proposée", command=ouvrir_corpus_Methode, width=30, height=2)
bouton3.place(x=250,y=300)

bouton4 = Button(fenetre, text="Méthode de Harrathi", command=ouvrir_corpus_nmethode, width=30, height=2)
bouton4.place(x=750,y=300)

bouton5 = Button(fenetre, text="Mesures", command=ouvrir_corpus_calcul, width=30, height=2)
bouton5.place(x=250,y=500)

bouton6 = Button(fenetre, text="Graphes", command=graphe_corpus, width=30, height=2)
bouton6.place(x=750,y=500)

bouton7 = Button(fenetre, text="Retour", command=retour , width=10, height=2)
bouton7.place(x=1200,y=500)

fenetre.mainloop()