h43z.js

const fs = require("fs");
const analysis = require("./analysis");
const Database = require("better-sqlite3");

exports.importH43zData = (sqliteFile, outputFile) => {
    const db = new Database(sqliteFile, { verbose: console.log });
    let items = [];
    const lookup = {};

    let stmt = db.prepare("select * from product");
    for (const row of stmt.iterate()) {
        const item = {
            store: row.shop == "billa" ? "billa" : "spar",
            id: row.product_id,
            name: row.name,
            price: 0,
            priceHistory: [],
            isWeighted: false,
            unit: "g",
            quantity: 0,
            bio: false,
            url: "",
        };
        items.push(item);
        lookup[row.product_id] = item;
    }
    console.log(items.length + " products");

    stmt = db.prepare("select * from pricehistory order by date asc");
    let i = 0;
    let unknown = 0;
    for (const row of stmt.iterate()) {
        i++;
        const item = lookup[row.product_id];
        if (!item) {
            unknown++;
        } else {
            if (item.priceHistory.length == 0 || item.priceHistory[item.priceHistory.length - 1].price != row.price)
                item.priceHistory.push({ date: row.date, price: row.price });
        }
        if (i % 1000 == 0) {
            console.log("Processed " + i + " prices, " + unknown + " unknown");
        }
    }

    items = items.filter((item) => item.priceHistory.length > 0);
    items.forEach((item) => {
        item.priceHistory.reverse();
        item.price = item.priceHistory[0];
    });
    analysis.writeJSON(outputFile, items);
};

exports.mergeWithLatestCanonical = (h43zFile, latestCanonicalFile) => {
    const items = analysis.readJSON(h43zFile);
    const lookup = {};
    items.forEach((item) => {
        // item.priceHistory = item.priceHistory.filter(price => price.date > "2020-01-01")
        lookup[item.id] = item;
    });
    const currItems = analysis.readJSON(latestCanonicalFile + "." + analysis.FILE_COMPRESSOR);
    const currLookup = {};
    currItems.forEach((item) => (currLookup[item.store + item.id] = item));
    let missingItems = {
        spar: 0,
        billa: 0,
    };
    let foundItems = {
        spar: 0,
        billa: 0,
    };
    for (item of items) {
        const i = lookup[item.id];
        const currItem = currLookup[item.store + item.id];
        if (!currItem) {
            missingItems[item.store]++;
        } else {
            foundItems[item.store]++;
            const oldHistory = [...currItem.priceHistory];
            currItem.priceHistory.push(...item.priceHistory);
            currItem.priceHistory.sort((a, b) => new Date(a.date) - new Date(b.date));

            const mergedHistory = [];
            currItem.priceHistory.forEach((price) => {
                if (mergedHistory.length == 0) {
                    mergedHistory.push(price);
                    return;
                }
                if (mergedHistory[mergedHistory.length - 1].price != price.price) {
                    mergedHistory.push(price);
                }
            });
            mergedHistory.reverse();
            currItem.priceHistory = mergedHistory;
        }
    }
    console.log(JSON.stringify(missingItems, null, 2));
    console.log(JSON.stringify(foundItems, null, 2));
    analysis.writeJSON(latestCanonicalFile, currItems, analysis.FILE_COMPRESSOR);
};

if (require.main === module) {
    exports.importH43zData("/Users/badlogic/Downloads/shops.db", "h43z.json");
    exports.mergeWithLatestCanonical("h43z.json", "data/latest-canonical.json");
}