From 03cf179d0256622c436de3489021b29ffaf197b7 Mon Sep 17 00:00:00 2001 From: SaekiTominaga Date: Sat, 8 Jul 2023 21:35:04 +0900 Subject: [PATCH 1/3] =?UTF-8?q?=E3=82=B5=E3=82=A4=E3=82=BA=E3=80=81?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=97=A5=E6=99=82=E3=81=AE=E5=88=97=E3=82=92?= =?UTF-8?q?=E5=BB=83=E6=AD=A2=E3=80=81=E3=83=8F=E3=83=83=E3=82=B7=E3=83=A5?= =?UTF-8?q?=E5=80=A4=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/admin/crawler-resource.ejs | 10 ++-------- packages/backend/node/@types/view.d.ts | 3 +-- .../node/src/controller/CrawlerResourceController.ts | 3 +-- packages/backend/node/src/dao/CrawlerResourceDao.ts | 10 +++------- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/html/admin/crawler-resource.ejs b/html/admin/crawler-resource.ejs index 72a1fec83..ca2317bac 100644 --- a/html/admin/crawler-resource.ejs +++ b/html/admin/crawler-resource.ejs @@ -124,8 +124,7 @@ 優先 🌐 セレクター - サイズ - 更新日時 + ハッシュ値 @@ -148,12 +147,7 @@ <%= resourcePage.priority %> <%_ if (resourcePage.browser) { _%>✔<%_ } _%> <%= resourcePage.selector %> - <%= resourcePage.content_length %> - <%_ if (resourcePage.last_modified !== null) { _%> - <%= resourcePage.last_modified.format('YYYY年M月D日 H時m分') %> - <%_ } else { _%> - - <%_ } _%> + <%= resourcePage.content_hash %> <%_ } _%> diff --git a/packages/backend/node/@types/view.d.ts b/packages/backend/node/@types/view.d.ts index 20e8f6546..38aaf7d3a 100644 --- a/packages/backend/node/@types/view.d.ts +++ b/packages/backend/node/@types/view.d.ts @@ -60,8 +60,7 @@ declare namespace CrawlerResourceView { priority: string; browser: boolean; selector: string | null; - content_length: number; - last_modified: import('dayjs').Dayjs | null; + content_hash: string; } } diff --git a/packages/backend/node/src/controller/CrawlerResourceController.ts b/packages/backend/node/src/controller/CrawlerResourceController.ts index 1db04ab0b..3a0795b30 100644 --- a/packages/backend/node/src/controller/CrawlerResourceController.ts +++ b/packages/backend/node/src/controller/CrawlerResourceController.ts @@ -132,8 +132,7 @@ export default class CrawlerResourceController extends Controller implements Con priority: resoursePage.priority, browser: resoursePage.browser, selector: resoursePage.selector, - content_length: resoursePage.content_length, - last_modified: resoursePage.last_modified, + content_hash: resoursePage.content_hash, }); resourcePageListView.set(categoryName, resourcePageOfCategoryView); diff --git a/packages/backend/node/src/dao/CrawlerResourceDao.ts b/packages/backend/node/src/dao/CrawlerResourceDao.ts index 4154dffda..8d9d617dd 100644 --- a/packages/backend/node/src/dao/CrawlerResourceDao.ts +++ b/packages/backend/node/src/dao/CrawlerResourceDao.ts @@ -1,4 +1,3 @@ -import dayjs, { Dayjs } from 'dayjs'; import CrawlerDao from './CrawlerDao.js'; import DbUtil from '../util/DbUtil.js'; @@ -9,8 +8,7 @@ interface ResourcePage { priority: string; browser: boolean; selector: string | null; - content_length: number; - last_modified: Dayjs | null; + content_hash: string; } interface ReviseData { @@ -42,8 +40,7 @@ export default class CrawlerResourceDao extends CrawlerDao { p.name AS priority, r.browser AS browser, r.selector AS selector, - r.content_length AS content_length, - r.last_modified AS last_modified + r.content_hash AS content_hash FROM d_resource r, m_class c, @@ -68,8 +65,7 @@ export default class CrawlerResourceDao extends CrawlerDao { priority: row.priority, browser: Boolean(row.browser), selector: row.selector, - content_length: row.content_length, - last_modified: row.last_modified !== null ? dayjs.unix(row.last_modified) : null, + content_hash: row.content_hash, }); } From 4d7d0bfdc369dce18d1e0965e1ba592d38b81d22 Mon Sep 17 00:00:00 2001 From: SaekiTominaga Date: Sat, 8 Jul 2023 21:48:11 +0900 Subject: [PATCH 2/3] =?UTF-8?q?=E3=83=86=E3=83=BC=E3=83=96=E3=83=AB?= =?UTF-8?q?=E3=81=AE=E3=82=AB=E3=83=A9=E3=83=A0=E5=90=8D=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/backend/node/src/dao/CrawlerResourceDao.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/node/src/dao/CrawlerResourceDao.ts b/packages/backend/node/src/dao/CrawlerResourceDao.ts index 8d9d617dd..762a97583 100644 --- a/packages/backend/node/src/dao/CrawlerResourceDao.ts +++ b/packages/backend/node/src/dao/CrawlerResourceDao.ts @@ -46,7 +46,7 @@ export default class CrawlerResourceDao extends CrawlerDao { m_class c, m_priority p WHERE - r.class = c.fk AND + r.category = c.fk AND r.priority = p.fk ORDER BY c.sort, @@ -90,7 +90,7 @@ export default class CrawlerResourceDao extends CrawlerDao { const sth = await dbh.prepare(` INSERT INTO d_resource - (url, title, class, priority, browser, selector) + (url, title, category, priority, browser, selector) VALUES (:url, :title, :category, :priority, :browser, :selector) `); @@ -131,7 +131,7 @@ export default class CrawlerResourceDao extends CrawlerDao { d_resource SET title = :title, - class = :category, + category = :category, priority = :priority, browser = :browser, selector = :selector @@ -196,7 +196,7 @@ export default class CrawlerResourceDao extends CrawlerDao { const sth = await dbh.prepare(` SELECT title, - class AS category, + category, priority, browser, selector From d1b3c203565b99319c4160ec3819afefaa60f245 Mon Sep 17 00:00:00 2001 From: SaekiTominaga Date: Sat, 8 Jul 2023 21:58:57 +0900 Subject: [PATCH 3/3] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E5=88=97?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- html/admin/crawler-resource.ejs | 6 ++++-- packages/backend/node/@types/view.d.ts | 1 + .../node/src/controller/CrawlerResourceController.ts | 1 + packages/backend/node/src/dao/CrawlerResourceDao.ts | 5 ++++- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/html/admin/crawler-resource.ejs b/html/admin/crawler-resource.ejs index ca2317bac..56eb3f6c2 100644 --- a/html/admin/crawler-resource.ejs +++ b/html/admin/crawler-resource.ejs @@ -125,6 +125,7 @@ 🌐 セレクター ハッシュ値 + エラー @@ -144,10 +145,11 @@ <%= resourcePage.title %>

<%= resourcePage.url %>

- <%= resourcePage.priority %> - <%_ if (resourcePage.browser) { _%>✔<%_ } _%> + <%= resourcePage.priority %> + <%_ if (resourcePage.browser) { _%>✔<%_ } _%> <%= resourcePage.selector %> <%= resourcePage.content_hash %> + <%_ if (resourcePage.error) { _%>✗<%_ } _%> <%_ } _%> diff --git a/packages/backend/node/@types/view.d.ts b/packages/backend/node/@types/view.d.ts index 38aaf7d3a..51a56fb80 100644 --- a/packages/backend/node/@types/view.d.ts +++ b/packages/backend/node/@types/view.d.ts @@ -61,6 +61,7 @@ declare namespace CrawlerResourceView { browser: boolean; selector: string | null; content_hash: string; + error: boolean; } } diff --git a/packages/backend/node/src/controller/CrawlerResourceController.ts b/packages/backend/node/src/controller/CrawlerResourceController.ts index 3a0795b30..c96ee7b03 100644 --- a/packages/backend/node/src/controller/CrawlerResourceController.ts +++ b/packages/backend/node/src/controller/CrawlerResourceController.ts @@ -133,6 +133,7 @@ export default class CrawlerResourceController extends Controller implements Con browser: resoursePage.browser, selector: resoursePage.selector, content_hash: resoursePage.content_hash, + error: resoursePage.error, }); resourcePageListView.set(categoryName, resourcePageOfCategoryView); diff --git a/packages/backend/node/src/dao/CrawlerResourceDao.ts b/packages/backend/node/src/dao/CrawlerResourceDao.ts index 762a97583..8e19232a7 100644 --- a/packages/backend/node/src/dao/CrawlerResourceDao.ts +++ b/packages/backend/node/src/dao/CrawlerResourceDao.ts @@ -9,6 +9,7 @@ interface ResourcePage { browser: boolean; selector: string | null; content_hash: string; + error: boolean; } interface ReviseData { @@ -40,7 +41,8 @@ export default class CrawlerResourceDao extends CrawlerDao { p.name AS priority, r.browser AS browser, r.selector AS selector, - r.content_hash AS content_hash + r.content_hash AS content_hash, + r.error AS error FROM d_resource r, m_class c, @@ -66,6 +68,7 @@ export default class CrawlerResourceDao extends CrawlerDao { browser: Boolean(row.browser), selector: row.selector, content_hash: row.content_hash, + error: Boolean(row.error), }); }