diff --git a/manifest.json b/manifest.json index 7eaff78..513ee7b 100644 --- a/manifest.json +++ b/manifest.json @@ -3,7 +3,7 @@ "name": "JDN", "description": "", "devtools_page": "index.html", - "version": "3.0.33", + "version": "3.0.34", "permissions": [ "activeTab", "tabs", diff --git a/package.json b/package.json index 05f85aa..55eb0f4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jdi-react-extension", - "version": "3.0.33", + "version": "3.0.34", "description": "jdi react extension", "scripts": { "start": "npm run webpack", @@ -42,6 +42,7 @@ "classnames": "^2.2.6", "cyrillic-to-translit-js": "^3.1.0", "jszip": "^3.6.0", + "lodash": "^4.17.21", "mobx": "^6.1.8", "mobx-react": "^7.1.0", "mobx-react-devtools": "^6.1.1", diff --git a/src/js/blocks/autoFind/autoFindProvider/AutoFindProvider.jsx b/src/js/blocks/autoFind/autoFindProvider/AutoFindProvider.jsx index 48a5569..092aed0 100644 --- a/src/js/blocks/autoFind/autoFindProvider/AutoFindProvider.jsx +++ b/src/js/blocks/autoFind/autoFindProvider/AutoFindProvider.jsx @@ -1,4 +1,5 @@ /* eslint-disable indent */ +import _ from "lodash"; import React, { useState, useEffect } from "react"; import { inject, observer } from "mobx-react"; import { useContext } from "react"; @@ -7,7 +8,7 @@ import { highlightElements, runDocumentListeners, generatePageObject, - requestXpathes, + requestGenerationData, } from "./pageDataHandlers"; import { reportProblemPopup } from "./contentScripts/reportProblemPopup/reportProblemPopup"; import { JDIclasses, getJdiClassName } from "./generationClassesMap"; @@ -103,7 +104,7 @@ const AutoFindProvider = inject("mainModel")( const reportProblem = (predictedElements) => { chrome.storage.sync.set( - { predictedElements }, + { predictedElements }, connector.attachContentScript(reportProblemPopup) ); }; @@ -163,12 +164,16 @@ const AutoFindProvider = inject("mainModel")( const onHighlighted = () => { setStatus(autoFindStatus.success); setAvailableForGeneration( - predictedElements.filter( - (e) => - e.predicted_probability >= perception && - !e.skipGeneration && - !e.hidden && - !unreachableNodes.includes(e.element_id) + _.unionBy( + availableForGeneration, + predictedElements.filter( + (e) => + e.predicted_probability >= perception && + !e.skipGeneration && + !e.hidden && + !unreachableNodes.includes(e.element_id) + ), + 'element_id' ) ); } @@ -195,21 +200,17 @@ const AutoFindProvider = inject("mainModel")( ); if (!noXpath.length) return; setXpathStatus(xpathGenerationStatus.started); - requestXpathes(noXpath, ({ xpathElements, unreachableNodes }) => { - setAvailableForGeneration(xpathElements); + requestGenerationData(noXpath, ({ generationData, unreachableNodes }) => { + setAvailableForGeneration( + _.chain(availableForGeneration) + .map((el) => _.chain(generationData).find({ element_id: el.element_id }).merge(el).value()) + .differenceBy(unreachableNodes, 'element_id') + .value() + ); setUnreachableNodes(unreachableNodes); - const updated = predictedElements.map((predictedElement) => { - const xPathEl = xpathElements.find( - (x) => x.element_id === predictedElement.element_id - ); - return { - ...predictedElement, - ...xPathEl, - }; - }); - setPredictedElements(updated); setXpathStatus(xpathGenerationStatus.complete); }); + }, [availableForGeneration]); useEffect(() => { diff --git a/src/js/blocks/autoFind/autoFindProvider/connector.js b/src/js/blocks/autoFind/autoFindProvider/connector.js index ec3d284..1a82151 100644 --- a/src/js/blocks/autoFind/autoFindProvider/connector.js +++ b/src/js/blocks/autoFind/autoFindProvider/connector.js @@ -109,8 +109,8 @@ export const sendMessage = { setHighlight: (payload) => connector.sendMessage("SET_HIGHLIGHT", payload), killHighlight: (payload, onResponse) => connector.sendMessage("KILL_HIGHLIGHT", null, onResponse), - generateXpathes: (payload, onResponse) => - connector.sendMessage("GENERATE_XPATHES", payload, onResponse), + generateAttributes: (payload, onResponse) => + connector.sendMessage("GENERATE_ATTRIBUTES", payload, onResponse), pingScript: (payload, onResponse) => connector.sendMessage("PING_SCRIPT", payload, onResponse), highlightUnreached: (payload) => connector.sendMessage("HIGHLIGHT_ERRORS", payload), diff --git a/src/js/blocks/autoFind/autoFindProvider/contentScripts/generationData.js b/src/js/blocks/autoFind/autoFindProvider/contentScripts/generationData.js index 87a915b..afece1c 100644 --- a/src/js/blocks/autoFind/autoFindProvider/contentScripts/generationData.js +++ b/src/js/blocks/autoFind/autoFindProvider/contentScripts/generationData.js @@ -1,388 +1,4 @@ -export const generateXpathes = () => { - let exports = {}; - /** - * Main class, containing the Algorithm. - * - * @remarks For more information on how the algorithm works, please refer to: - * Maurizio Leotta, Andrea Stocco, Filippo Ricca, Paolo Tonella. ROBULA+: - * An Algorithm for Generating Robust XPath Locators for Web Testing. Journal - * of Software: Evolution and Process (JSEP), Volume 28, Issue 3, pp.177–204. - * John Wiley & Sons, 2016. - * https://doi.org/10.1002/smr.1771 - * - * @param options - (optional) algorithm options. - */ - class RobulaPlus { - constructor(options) { - this.attributePriorizationList = [ - "name", - "class", - "title", - "alt", - "value", - ]; - this.attributeBlackList = [ - "href", - "src", - "onclick", - "onload", - "tabindex", - "width", - "height", - "style", - "size", - "maxlength", - "jdn-hash", - "xml:space", - ]; - if (options) { - this.attributePriorizationList = options.attributePriorizationList; - this.attributeBlackList = options.attributeBlackList; - } - } - /** - * Returns an optimized robust XPath locator string. - * - * @param element - The desired element. - * @param document - The document to analyse, that contains the desired element. - * - * @returns - A robust xPath locator string, describing the desired element. - */ - getRobustXPath(element, document) { - if (!document.body.contains(element)) { - throw new Error("Document does not contain given element!"); - } - const xPathList = [new XPath("//*")]; - while (xPathList.length > 0) { - const xPath = xPathList.shift(); - let temp = []; - temp = temp.concat(this.transfConvertStar(xPath, element)); - temp = temp.concat(this.transfAddId(xPath, element)); - temp = temp.concat(this.transfAddText(xPath, element)); - temp = temp.concat(this.transfAddAttribute(xPath, element)); - temp = temp.concat(this.transfAddAttributeSet(xPath, element)); - temp = temp.concat(this.transfAddPosition(xPath, element)); - temp = temp.concat(this.transfAddLevel(xPath, element)); - temp = [...new Set(temp)]; // removes duplicates - for (const x of temp) { - if (this.uniquelyLocate(x.getValue(), element, document)) { - return x.getValue(); - } - xPathList.push(x); - } - } - throw new Error("Internal Error: xPathList.shift returns undefined"); - } - /** - * Returns an element in the given document located by the given xPath locator. - * - * @param xPath - A xPath string, describing the desired element. - * @param document - The document to analyse, that contains the desired element. - * - * @returns - The first maching Element located. - */ - getElementByXPath(xPath, document) { - return document.evaluate( - xPath, - document, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - null - ).singleNodeValue; - } - /** - * Returns, wheater an xPath describes only the given element. - * - * @param xPath - A xPath string, describing the desired element. - * @param element - The desired element. - * @param document - The document to analyse, that contains the desired element. - * - * @returns - True, if the xPath describes only the desired element. - */ - uniquelyLocate(xPath, element, document) { - const nodesSnapshot = document.evaluate( - xPath, - document, - null, - XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, - null - ); - return ( - nodesSnapshot.snapshotLength === 1 && - nodesSnapshot.snapshotItem(0) === element - ); - } - transfConvertStar(xPath, element) { - const output = []; - const ancestor = this.getAncestor(element, xPath.getLength() - 1); - if (xPath.startsWith("//*")) { - output.push( - new XPath("//" + ancestor.tagName.toLowerCase() + xPath.substring(3)) - ); - } - return output; - } - transfAddId(xPath, element) { - const output = []; - const ancestor = this.getAncestor(element, xPath.getLength() - 1); - if (ancestor.id && !xPath.headHasAnyPredicates()) { - const newXPath = new XPath(xPath.getValue()); - newXPath.addPredicateToHead(`[@id='${ancestor.id}']`); - output.push(newXPath); - } - return output; - } - transfAddText(xPath, element) { - const output = []; - const ancestor = this.getAncestor(element, xPath.getLength() - 1); - if ( - ancestor.textContent && - !xPath.headHasPositionPredicate() && - !xPath.headHasTextPredicate() - ) { - const text = ancestor.textContent.replace(/\'/g, """).trim(); - const newXPath = new XPath(xPath.getValue()); - newXPath.addPredicateToHead(`[contains(text(),'${text}')]`); - output.push(newXPath); - } - return output; - } - transfAddAttribute(xPath, element) { - const output = []; - const ancestor = this.getAncestor(element, xPath.getLength() - 1); - if (!xPath.headHasAnyPredicates()) { - // add priority attributes to output - for (const priorityAttribute of this.attributePriorizationList) { - for (const attribute of ancestor.attributes) { - if (attribute.name === priorityAttribute) { - const newXPath = new XPath(xPath.getValue()); - newXPath.addPredicateToHead( - `[@${attribute.name}='${attribute.value}']` - ); - output.push(newXPath); - break; - } - } - } - // append all other non-blacklist attributes to output - for (const attribute of ancestor.attributes) { - if ( - !this.attributeBlackList.includes(attribute.name) && - !this.attributePriorizationList.includes(attribute.name) - ) { - const newXPath = new XPath(xPath.getValue()); - newXPath.addPredicateToHead( - `[@${attribute.name}='${attribute.value}']` - ); - output.push(newXPath); - } - } - } - return output; - } - transfAddAttributeSet(xPath, element) { - const output = []; - const ancestor = this.getAncestor(element, xPath.getLength() - 1); - if (!xPath.headHasAnyPredicates()) { - // add id to attributePriorizationList - this.attributePriorizationList.unshift("id"); - let attributes = [...ancestor.attributes]; - // remove black list attributes - attributes = attributes.filter( - (attribute) => !this.attributeBlackList.includes(attribute.name) - ); - // generate power set - let attributePowerSet = this.generatePowerSet(attributes); - // remove sets with cardinality < 2 - attributePowerSet = attributePowerSet.filter( - (attributeSet) => attributeSet.length >= 2 - ); - // sort elements inside each powerset - for (const attributeSet of attributePowerSet) { - attributeSet.sort(this.elementCompareFunction.bind(this)); - } - // sort attributePowerSet - attributePowerSet.sort((set1, set2) => { - if (set1.length < set2.length) { - return -1; - } - if (set1.length > set2.length) { - return 1; - } - for (let i = 0; i < set1.length; i++) { - if (set1[i] !== set2[i]) { - return this.elementCompareFunction(set1[i], set2[i]); - } - } - return 0; - }); - // remove id from attributePriorizationList - this.attributePriorizationList.shift(); - // convert to predicate - for (const attributeSet of attributePowerSet) { - let predicate = `[@${attributeSet[0].name}='${attributeSet[0].value}'`; - for (let i = 1; i < attributeSet.length; i++) { - predicate += ` and @${attributeSet[i].name}='${attributeSet[i].value}'`; - } - predicate += "]"; - const newXPath = new XPath(xPath.getValue()); - newXPath.addPredicateToHead(predicate); - output.push(newXPath); - } - } - return output; - } - transfAddPosition(xPath, element) { - const output = []; - const ancestor = this.getAncestor(element, xPath.getLength() - 1); - if (!xPath.headHasPositionPredicate()) { - let position = 1; - if (xPath.startsWith("//*")) { - position = - Array.from(ancestor.parentNode.children).indexOf(ancestor) + 1; - } else { - for (const child of ancestor.parentNode.children) { - if (ancestor === child) { - break; - } - if (ancestor.tagName === child.tagName) { - position++; - } - } - } - const newXPath = new XPath(xPath.getValue()); - newXPath.addPredicateToHead(`[${position}]`); - output.push(newXPath); - } - return output; - } - transfAddLevel(xPath, element) { - const output = []; - if (xPath.getLength() - 1 < this.getAncestorCount(element)) { - output.push(new XPath("//*" + xPath.substring(1))); - } - return output; - } - generatePowerSet(input) { - return input.reduce( - (subsets, value) => - subsets.concat(subsets.map((set) => [value, ...set])), - [[]] - ); - } - elementCompareFunction(attr1, attr2) { - for (const element of this.attributePriorizationList) { - if (element === attr1.name) { - return -1; - } - if (element === attr2.name) { - return 1; - } - } - return 0; - } - getAncestor(element, index) { - let output = element; - for (let i = 0; i < index; i++) { - if (!output) { - debugger; - console.log(output); - } - if (output.parentElement) { - output = output.parentElement; - } - } - return output; - } - getAncestorCount(element) { - let count = 0; - while (element.parentElement) { - element = element.parentElement; - count++; - } - return count; - } - } - exports.RobulaPlus = RobulaPlus; - class XPath { - constructor(value) { - this.value = value; - } - getValue() { - return this.value; - } - startsWith(value) { - return this.value.startsWith(value); - } - substring(value) { - return this.value.substring(value); - } - headHasAnyPredicates() { - return this.value.split("/")[2].includes("["); - } - headHasPositionPredicate() { - const splitXPath = this.value.split("/"); - const regExp = new RegExp("[[0-9]]"); - return ( - splitXPath[2].includes("position()") || - splitXPath[2].includes("last()") || - regExp.test(splitXPath[2]) - ); - } - headHasTextPredicate() { - return this.value.split("/")[2].includes("text()"); - } - addPredicateToHead(predicate) { - const splitXPath = this.value.split("/"); - splitXPath[2] += predicate; - this.value = splitXPath.join("/"); - } - getLength() { - const splitXPath = this.value.split("/"); - let length = 0; - for (const piece of splitXPath) { - if (piece) { - length++; - } - } - return length; - } - } - exports.XPath = XPath; - class RobulaPlusOptions { - constructor() { - /** - * @attribute - attributePriorizationList: A prioritized list of HTML attributes, which are considered in the given order. - * @attribute - attributeBlackList: Contains HTML attributes, which are classified as too fragile and are ignored by the algorithm. - */ - this.attributePriorizationList = [ - "name", - "class", - "title", - "alt", - "value", - ]; - this.attributeBlackList = [ - "href", - "src", - "onclick", - "onload", - "tabindex", - "width", - "height", - "style", - "size", - "maxlength", - ]; - } - } - exports.RobulaPlusOptions = RobulaPlusOptions; - - window.robula = exports; - - const unreachableNodes = []; - const robula = new RobulaPlus(); - +export const getGenerationAttributes = () => { /* Make an 'ID' attribute to the camel notation. Rules: - Replace the dash just before the letters (search-button -> searchButton) @@ -402,12 +18,11 @@ export const generateXpathes = () => { }; const mapElements = (elements) => { - const xpathElements = (elements.map((predictedElement, index) => { + const generationAttributes = (elements.map((predictedElement) => { let element = document.querySelector( `[jdn-hash='${predictedElement.element_id}']` ); if (!element) { - unreachableNodes.push(predictedElement.element_id); return; } predictedElement.attrId = element.id; @@ -416,21 +31,15 @@ export const generateXpathes = () => { : ""; predictedElement.tagName = element.tagName.toLowerCase(); - console.log(`${index + 1} xpathes been generated`); - return { ...predictedElement, - xpath: robula.getRobustXPath(element, document), }; })).filter(el => !!el); - return { - xpathElements, - unreachableNodes, - }; + return generationAttributes; }; chrome.runtime.onMessage.addListener(({ message, param }, sender, sendResponse) => { - if (message === "GENERATE_XPATHES") { + if (message === "GENERATE_ATTRIBUTES") { sendResponse(mapElements(param)); } diff --git a/src/js/blocks/autoFind/autoFindProvider/pageDataHandlers.js b/src/js/blocks/autoFind/autoFindProvider/pageDataHandlers.js index ccb2e53..4d08608 100644 --- a/src/js/blocks/autoFind/autoFindProvider/pageDataHandlers.js +++ b/src/js/blocks/autoFind/autoFindProvider/pageDataHandlers.js @@ -1,6 +1,6 @@ import { connector, sendMessage } from "./connector"; import { runContextMenu } from "./contentScripts/contextMenu/contextmenu"; -import { generateXpathes } from "./contentScripts/generationData"; +import { getGenerationAttributes } from "./contentScripts/generationData"; import { highlightOnPage } from "./contentScripts/highlight"; import { getPageData } from "./contentScripts/pageData"; import { urlListener } from "./contentScripts/urlListener"; @@ -55,7 +55,7 @@ const setUrlListener = (onHighlightOff) => { export const getElements = (callback, setStatus) => { const pageAccessTimeout = setTimeout(() => { setStatus(autoFindStatus.blocked); - }, 5000 ); + }, 5000); connector.updateMessageListener((payload) => { if (payload.message === "START_COLLECT_DATA") { @@ -66,11 +66,11 @@ export const getElements = (callback, setStatus) => { }); return connector.attachContentScript(getPageData) - .then(uploadElements) - .then((data) => { - removeOverlay(); - callback(data); - }); + .then(uploadElements) + .then((data) => { + removeOverlay(); + callback(data); + }); }; export const highlightElements = (elements, successCallback, perception) => { @@ -90,6 +90,21 @@ const messageHandler = ({ message, param }, actions) => { } }; +const requestGenerationAttributes = async (elements) => { + await connector.attachContentScript(getGenerationAttributes); + + return new Promise((resolve) => { + sendMessage.generateAttributes(elements, (response) => { + if (chrome.runtime.lastError) { + resolve(false); + } + if (response) { + resolve(response); + } else resolve(false); + }); + }); +}; + export const runDocumentListeners = (actions) => { connector.updateMessageListener((payload) => messageHandler(payload, actions) @@ -102,12 +117,45 @@ export const runDocumentListeners = (actions) => { } }; -export const requestXpathes = (elements, callback) => { - connector - .attachContentScript(generateXpathes) - .then(() => sendMessage.generateXpathes(elements, callback)); +export const requestXpathes = async (elements) => { + const documentResult = await connector.attachContentScript( + (() => JSON.stringify(document.documentElement.innerHTML)) + ); + + const document = await documentResult[0].result; + const ids = elements.map(el => el.element_id); + + const xPathResponse = await fetch("http:localhost:5000/generate_xpath", { + method: "POST", + body: JSON.stringify({ + ids, + document, + }), + }); + + if (xPathResponse.ok) { + const xPathes = await xPathResponse.json(); + const r = elements.map(el => ({ ...el, xpath: xPathes[el.element_id] })); + const unreachableNodes = r.filter(el => !el.xpath); + return { xpathes: r.filter(el => !!el.xpath), unreachableNodes }; + } else { + throw new Error(xPathResponse); + } }; +export const requestGenerationData = async (elements, callback) => { + const { xpathes, unreachableNodes } = await (await requestXpathes(elements)); + const generationAttributes = await requestGenerationAttributes(elements); + const generationData = xpathes.map(el => { + const attr = generationAttributes.find(g => g.element_id === el.element_id); + return { + ...el, + ...attr, + } + }); + callback({ generationData, unreachableNodes }); +} + export const generatePageObject = (elements, mainModel) => { const elToConvert = predictedToConvert(elements); getPage(elToConvert, (page) => {