Merge pull request #336 from SaekiTominaga/crawler-resource-hash

「ウェブ巡回（リソース）」の差分比較をコンテンツの長さ→ハッシュ値に変更
SaekiTominaga · Jul 8, 2023 · 5c27aad · 5c27aad
2 parents d1f0b6e + d1b3c20
commit 5c27aad
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 25 deletions.
diff --git a/html/admin/crawler-resource.ejs b/html/admin/crawler-resource.ejs
@@ -124,8 +124,8 @@
 				<th scope="col">優先</th>
 				<th scope="col">🌐</th>
 				<th scope="col">セレクター</th>
-				<th scope="col">サイズ</th>
-				<th scope="col">更新日時</th>
+				<th scope="col">ハッシュ値</th>
+				<th scope="col">エラー</th>
 			</tr>
 		</thead>
 		<tbody>
@@ -145,15 +145,11 @@
 					<a href="<%= resourcePage.url %>" referrerpolicy="no-referrer"><%= resourcePage.title %></a>
 					<p style="margin-block-start: 0.5em; font-size: 80%"><%= resourcePage.url %></p>
 				</td>
-				<td><%= resourcePage.priority %></td>
-				<td><%_ if (resourcePage.browser) { _%>✔<%_ } _%></td>
+				<td class="u-cell -center"><%= resourcePage.priority %></td>
+				<td class="u-cell -center"><%_ if (resourcePage.browser) { _%>✔<%_ } _%></td>
 				<td class="u-cell -wrap-anywhere"><%= resourcePage.selector %></td>
-				<td><%= resourcePage.content_length %></td>
-				<%_ if (resourcePage.last_modified !== null) { _%>
-				<td><%= resourcePage.last_modified.format('YYYY年M月D日 H時m分') %></td>
-				<%_ } else { _%>
-				<td></td>
-				<%_ } _%>
+				<td><%= resourcePage.content_hash %></td>
+				<td class="u-cell -center"><%_ if (resourcePage.error) { _%><strong style="color: red">✗</strong><%_ } _%></td>
 			</tr>
 			<%_ } _%>
 		</tbody>

diff --git a/packages/backend/node/@types/view.d.ts b/packages/backend/node/@types/view.d.ts
@@ -60,8 +60,8 @@ declare namespace CrawlerResourceView {
 		priority: string;
 		browser: boolean;
 		selector: string | null;
-		content_length: number;
-		last_modified: import('dayjs').Dayjs | null;
+		content_hash: string;
+		error: boolean;
 	}
 }
 

diff --git a/packages/backend/node/src/controller/CrawlerResourceController.ts b/packages/backend/node/src/controller/CrawlerResourceController.ts
@@ -132,8 +132,8 @@ export default class CrawlerResourceController extends Controller implements Con
 				priority: resoursePage.priority,
 				browser: resoursePage.browser,
 				selector: resoursePage.selector,
-				content_length: resoursePage.content_length,
-				last_modified: resoursePage.last_modified,
+				content_hash: resoursePage.content_hash,
+				error: resoursePage.error,
 			});
 
 			resourcePageListView.set(categoryName, resourcePageOfCategoryView);

diff --git a/packages/backend/node/src/dao/CrawlerResourceDao.ts b/packages/backend/node/src/dao/CrawlerResourceDao.ts
@@ -1,4 +1,3 @@
-import dayjs, { Dayjs } from 'dayjs';
 import CrawlerDao from './CrawlerDao.js';
 import DbUtil from '../util/DbUtil.js';
 
@@ -9,8 +8,8 @@ interface ResourcePage {
 	priority: string;
 	browser: boolean;
 	selector: string | null;
-	content_length: number;
-	last_modified: Dayjs | null;
+	content_hash: string;
+	error: boolean;
 }
 
 interface ReviseData {
@@ -42,14 +41,14 @@ export default class CrawlerResourceDao extends CrawlerDao {
 				p.name AS priority,
 				r.browser AS browser,
 				r.selector AS selector,
-				r.content_length AS content_length,
-				r.last_modified AS last_modified
+				r.content_hash AS content_hash,
+				r.error AS error
 			FROM
 				d_resource r,
 				m_class c,
 				m_priority p
 			WHERE
-				r.class = c.fk AND
+				r.category = c.fk AND
 				r.priority = p.fk
 			ORDER BY
 				c.sort,
@@ -68,8 +67,8 @@ export default class CrawlerResourceDao extends CrawlerDao {
 				priority: row.priority,
 				browser: Boolean(row.browser),
 				selector: row.selector,
-				content_length: row.content_length,
-				last_modified: row.last_modified !== null ? dayjs.unix(row.last_modified) : null,
+				content_hash: row.content_hash,
+				error: Boolean(row.error),
 			});
 		}
 
@@ -94,7 +93,7 @@ export default class CrawlerResourceDao extends CrawlerDao {
 			const sth = await dbh.prepare(`
 				INSERT INTO
 					d_resource
-					(url, title, class, priority, browser, selector)
+					(url, title, category, priority, browser, selector)
 				VALUES
 					(:url, :title, :category, :priority, :browser, :selector)
 			`);
@@ -135,7 +134,7 @@ export default class CrawlerResourceDao extends CrawlerDao {
 					d_resource
 				SET
 					title = :title,
-					class = :category,
+					category = :category,
 					priority = :priority,
 					browser = :browser,
 					selector = :selector
@@ -200,7 +199,7 @@ export default class CrawlerResourceDao extends CrawlerDao {
 		const sth = await dbh.prepare(`
 			SELECT
 				title,
-				class AS category,
+				category,
 				priority,
 				browser,
 				selector