Log checked headers for MCD events (#43)

* Log checked headers for Bite events

* Log checked headers for Drip drip events

* Log checked headers for Drip file events

- mark headers checked for 3 implementations of file on the Drip contract

* Log checked headers for Frob frob events

* Log checked headers for Pit file events

- mark headers checked for 3 implementations of file on the Pit contract

* Log checked headers for Vat init events

db/migrations/1538064962_add_bite_to_checked_headers.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN bite_checked;

db/migrations/1538064962_add_bite_to_checked_headers.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN bite_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538067790_add_drip_drip_to_checked_headers.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN drip_drip_checked;

db/migrations/1538067790_add_drip_drip_to_checked_headers.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN drip_drip_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538070524_add_drip_file_to_checked_headers.down.sql

@@ -0,0 +1,6 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN drip_file_ilk_checked;
+ALTER TABLE public.checked_headers
+    DROP COLUMN drip_file_repo_checked;
+ALTER TABLE public.checked_headers
+    DROP COLUMN drip_file_vow_checked;

db/migrations/1538070524_add_drip_file_to_checked_headers.up.sql

@@ -0,0 +1,6 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN drip_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE public.checked_headers
+    ADD COLUMN drip_file_repo_checked BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE public.checked_headers
+    ADD COLUMN drip_file_vow_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538075501_add_frob_to_checked_headers.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN frob_checked;

db/migrations/1538075501_add_frob_to_checked_headers.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN frob_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538078432_add_pit_file_to_checked_headers.down.sql

@@ -0,0 +1,6 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN pit_file_debt_ceiling_checked;
+ALTER TABLE public.checked_headers
+    DROP COLUMN pit_file_ilk_checked;
+ALTER TABLE public.checked_headers
+    DROP COLUMN pit_file_stability_fee_checked;

db/migrations/1538078432_add_pit_file_to_checked_headers.up.sql

@@ -0,0 +1,6 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN pit_file_debt_ceiling_checked BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE public.checked_headers
+    ADD COLUMN pit_file_ilk_checked BOOLEAN NOT NULL DEFAULT FALSE;
+ALTER TABLE public.checked_headers
+    ADD COLUMN pit_file_stability_fee_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/migrations/1538082658_add_vat_init_to_checked_headers.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    DROP COLUMN vat_init_checked;

db/migrations/1538082658_add_vat_init_to_checked_headers.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.checked_headers
+    ADD COLUMN vat_init_checked BOOLEAN NOT NULL DEFAULT FALSE;

db/schema.sql

@@ -820,7 +820,17 @@ CREATE TABLE public.checked_headers (
     tend_checked boolean DEFAULT false NOT NULL,
     cat_file_chop_lump_checked boolean DEFAULT false NOT NULL,
     cat_file_flip_checked boolean DEFAULT false NOT NULL,
-    cat_file_pit_vow_checked boolean DEFAULT false NOT NULL
+    cat_file_pit_vow_checked boolean DEFAULT false NOT NULL,
+    bite_checked boolean DEFAULT false NOT NULL,
+    drip_drip_checked boolean DEFAULT false NOT NULL,
+    drip_file_ilk_checked boolean DEFAULT false NOT NULL,
+    drip_file_repo_checked boolean DEFAULT false NOT NULL,
+    drip_file_vow_checked boolean DEFAULT false NOT NULL,
+    frob_checked boolean DEFAULT false NOT NULL,
+    pit_file_debt_ceiling_checked boolean DEFAULT false NOT NULL,
+    pit_file_ilk_checked boolean DEFAULT false NOT NULL,
+    pit_file_stability_fee_checked boolean DEFAULT false NOT NULL,
+    vat_init_checked boolean DEFAULT false NOT NULL
 );
 
 

pkg/transformers/bite/converter.go

@@ -19,66 +19,75 @@ package bite
 import (
 	"encoding/json"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/vulcanize/vulcanizedb/pkg/geth"
 	"github.com/vulcanize/vulcanizedb/pkg/transformers/shared"
 )
 
 type Converter interface {
-	ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error)
-	ToModel(flipKick BiteEntity) (BiteModel, error)
+	ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error)
+	ToModels(biteEntities []BiteEntity) ([]BiteModel, error)
 }
 
 type BiteConverter struct{}
 
-func (BiteConverter) ToEntity(contractAddress string, contractAbi string, ethLog types.Log) (BiteEntity, error) {
-	entity := BiteEntity{}
-	address := common.HexToAddress(contractAddress)
-	abi, err := geth.ParseAbi(contractAbi)
-	if err != nil {
-		return entity, err
-	}
+func (BiteConverter) ToEntities(contractAbi string, ethLogs []types.Log) ([]BiteEntity, error) {
+	var entities []BiteEntity
+	for _, ethLog := range ethLogs {
+		entity := BiteEntity{}
+		address := ethLog.Address
+		abi, err := geth.ParseAbi(contractAbi)
+		if err != nil {
+			return nil, err
+		}
 
-	contract := bind.NewBoundContract(address, abi, nil, nil, nil)
+		contract := bind.NewBoundContract(address, abi, nil, nil, nil)
 
-	err = contract.UnpackLog(&entity, "Bite", ethLog)
-	if err != nil {
-		return entity, err
-	}
+		err = contract.UnpackLog(&entity, "Bite", ethLog)
+		if err != nil {
+			return nil, err
+		}
+
+		entity.Raw = ethLog
+		entity.TransactionIndex = ethLog.TxIndex
 
-	entity.Raw = ethLog
-	entity.TransactionIndex = ethLog.TxIndex
+		entities = append(entities, entity)
+	}
 
-	return entity, nil
+	return entities, nil
 }
-func (converter BiteConverter) ToModel(entity BiteEntity) (BiteModel, error) {
 
-	id := entity.Id
-	ilk := entity.Ilk[:]
-	urn := entity.Urn[:]
-	ink := entity.Ink
-	art := entity.Art
-	iArt := entity.IArt
-	tab := entity.Tab
-	flip := entity.Flip
-	txIdx := entity.TransactionIndex
-	rawLogJson, err := json.Marshal(entity.Raw)
-	rawLogString := string(rawLogJson)
-	if err != nil {
-		return BiteModel{}, err
-	}
+func (converter BiteConverter) ToModels(entities []BiteEntity) ([]BiteModel, error) {
+	var models []BiteModel
+	for _, entity := range entities {
+		id := entity.Id
+		ilk := entity.Ilk[:]
+		urn := entity.Urn[:]
+		ink := entity.Ink
+		art := entity.Art
+		iArt := entity.IArt
+		tab := entity.Tab
+		flip := entity.Flip
+		txIdx := entity.TransactionIndex
+		rawLogJson, err := json.Marshal(entity.Raw)
+		rawLogString := string(rawLogJson)
+		if err != nil {
+			return nil, err
+		}
 
-	return BiteModel{
-		Id:               shared.BigIntToString(id),
-		Ilk:              ilk,
-		Urn:              urn,
-		Ink:              shared.BigIntToString(ink),
-		Art:              shared.BigIntToString(art),
-		IArt:             shared.BigIntToString(iArt),
-		Tab:              shared.BigIntToString(tab),
-		Flip:             shared.BigIntToString(flip),
-		TransactionIndex: txIdx,
-		Raw:              rawLogString,
-	}, nil
+		model := BiteModel{
+			Id:               shared.BigIntToString(id),
+			Ilk:              ilk,
+			Urn:              urn,
+			Ink:              shared.BigIntToString(ink),
+			Art:              shared.BigIntToString(art),
+			IArt:             shared.BigIntToString(iArt),
+			Tab:              shared.BigIntToString(tab),
+			Flip:             shared.BigIntToString(flip),
+			TransactionIndex: txIdx,
+			Raw:              rawLogString,
+		}
+		models = append(models, model)
+	}
+	return models, nil
 }

pkg/transformers/bite/converter_test.go

@@ -34,9 +34,11 @@ var _ = Describe("Bite Converter", func() {
 
 	Describe("ToEntity", func() {
 		It("converts an eth log to a bite entity", func() {
-			entity, err := converter.ToEntity(shared.CatContractAddress, shared.CatABI, test_data.EthBiteLog)
+			entities, err := converter.ToEntities(shared.CatABI, []types.Log{test_data.EthBiteLog})
 
 			Expect(err).NotTo(HaveOccurred())
+			Expect(len(entities)).To(Equal(1))
+			entity := entities[0]
 			Expect(entity.Ilk).To(Equal(test_data.BiteEntity.Ilk))
 			Expect(entity.Urn).To(Equal(test_data.BiteEntity.Urn))
 			Expect(entity.Ink).To(Equal(test_data.BiteEntity.Ink))
@@ -49,7 +51,7 @@ var _ = Describe("Bite Converter", func() {
 		})
 
 		It("returns an error if converting log to entity fails", func() {
-			_, err := converter.ToEntity(shared.CatContractAddress, "error abi", test_data.EthBiteLog)
+			_, err := converter.ToEntities("error abi", []types.Log{test_data.EthBiteLog})
 
 			Expect(err).To(HaveOccurred())
 		})
@@ -63,9 +65,11 @@ var _ = Describe("Bite Converter", func() {
 		})
 
 		It("converts an Entity to a Model", func() {
-			model, err := converter.ToModel(test_data.BiteEntity)
+			models, err := converter.ToModels([]bite.BiteEntity{test_data.BiteEntity})
 
 			Expect(err).NotTo(HaveOccurred())
+			Expect(len(models)).To(Equal(1))
+			model := models[0]
 			Expect(model).To(Equal(test_data.BiteModel))
 			Expect(model.TransactionIndex).To(Equal(test_data.BiteModel.TransactionIndex))
 		})
@@ -85,9 +89,11 @@ var _ = Describe("Bite Converter", func() {
 				TransactionIndex: 0,
 				Raw:              string(emptyLog),
 			}
-			model, err := converter.ToModel(emptyEntity)
+			models, err := converter.ToModels([]bite.BiteEntity{emptyEntity})
 
 			Expect(err).NotTo(HaveOccurred())
+			Expect(len(models)).To(Equal(1))
+			model := models[0]
 			Expect(model).To(Equal(expectedModel))
 		})
 	})

pkg/transformers/bite/repository.go

@@ -20,7 +20,8 @@ import (
 )
 
 type Repository interface {
-	Create(headerID int64, model BiteModel) error
+	Create(headerID int64, models []BiteModel) error
+	MarkHeaderChecked(headerID int64) error
 	MissingHeaders(startingBlockNumber, endingBlockNumber int64) ([]core.Header, error)
 }
 
@@ -32,26 +33,48 @@ func NewBiteRepository(db *postgres.DB) Repository {
 	return BiteRepository{db: db}
 }
 
-func (repository BiteRepository) Create(headerID int64, model BiteModel) error {
-	_, err := repository.db.Exec(
-		`INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log)
+func (repository BiteRepository) Create(headerID int64, models []BiteModel) error {
+	tx, err := repository.db.Begin()
+	if err != nil {
+		return err
+	}
+	for _, model := range models {
+		_, err := tx.Exec(
+			`INSERT into maker.bite (header_id, id, ilk, urn, ink, art, iart, tab, flip, tx_idx, raw_log)
         VALUES($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`,
-		headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw,
-	)
-
+			headerID, model.Id, model.Ilk, model.Urn, model.Ink, model.Art, model.IArt, model.Tab, model.Flip, model.TransactionIndex, model.Raw,
+		)
+		if err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+	_, err = tx.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET bite_checked = $2`, headerID, true)
 	if err != nil {
+		tx.Rollback()
 		return err
 	}
+	return tx.Commit()
+}
 
-	return nil
+func (repository BiteRepository) MarkHeaderChecked(headerID int64) error {
+	_, err := repository.db.Exec(`INSERT INTO public.checked_headers (header_id, bite_checked)
+		VALUES ($1, $2) 
+	ON CONFLICT (header_id) DO
+		UPDATE SET bite_checked = $2`, headerID, true)
+	return err
 }
+
 func (repository BiteRepository) MissingHeaders(startingBlockNumber int64, endingBlockNumber int64) ([]core.Header, error) {
 	var result []core.Header
 	err := repository.db.Select(
 		&result,
 		`SELECT headers.id, headers.block_number FROM headers
-               LEFT JOIN maker.bite on headers.id = header_id
-               WHERE header_id ISNULL
+               LEFT JOIN checked_headers on headers.id = header_id
+               WHERE (header_id ISNULL OR bite_checked IS FALSE)
                AND headers.block_number >= $1
                AND headers.block_number <= $2
                AND headers.eth_node_fingerprint = $3`,