diff --git a/adserver/crawler/crawler.go b/adserver/crawler/crawler.go
new file mode 100644
index 0000000..87a6d71
--- /dev/null
+++ b/adserver/crawler/crawler.go
@@ -0,0 +1,192 @@
+package crawler
+
+import (
+ "YellowBloomKnapsack/mini-yektanet/adserver/kvstorage"
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "net/http"
+ "os"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "golang.org/x/net/html"
+)
+
+type Crawler struct {
+ kvstorage kvstorage.KVStorageInterface
+ persianStopWords map[string]bool
+ numOftopwords int
+ urls map[string]int
+}
+
+// Publisher IDs.
+var publisherIDs = map[string]int{
+ "varzesh3": 1,
+ "digikala": 2,
+ "zoomit": 3,
+ "sheypoor": 4,
+ "filimo": 5,
+}
+
+func NewCrawler(kvstorage kvstorage.KVStorageInterface) *Crawler {
+ baseUrl := "http://" + os.Getenv("PUBLISHER_WEBSITE_HOSTNAME") + ":" + os.Getenv("PUBLISHER_WEBSITE_PORT")
+ return &Crawler{
+ kvstorage: kvstorage,
+ persianStopWords: map[string]bool{
+ "و": true, "در": true, "به": true, "از": true, "که": true, "این": true, "را": true, "اینجا": true,
+ "با": true, "برای": true, "است": true, "آن": true, "یک": true, "تا": true, "هم": true, "کنیم": true,
+ "می": true, "بر": true, "بود": true, "شد": true, "یا": true, "وی": true, "اما": true, "داریم": true, "اولین": true,
+ "اگر": true, "هر": true, "من": true, "ما": true, "شما": true, "او": true, "آنها": true, "دهیم": true, "آخرین": true,
+ "ایشان": true, "بودن": true, "باشند": true, "نیز": true, "چون": true, "چه": true, "نیست": true, "های": true,
+ "هیچ": true, "همین": true, "چیزی": true, "دارند": true, "کنند": true, "خواهد": true, "آیا": true, "ها": true,
+ "کنید": true, "بدانید": true, "خوش": true, "آمدید": true, "خود": true, "زیاد": true, "کم": true, "زیادی": true,
+ },
+ numOftopwords: 5,
+ urls: map[string]int{
+ baseUrl + "/varzesh3": 1,
+ baseUrl + "/digikala": 2,
+ baseUrl + "/zoomit": 3,
+ baseUrl + "/sheypoor": 4,
+ baseUrl + "/filimo": 5,
+ },
+ }
+}
+
+func (c *Crawler) readHTML(url string) ([]byte, error) {
+ resp, err := http.Get(url)
+ if err != nil {
+ return nil, fmt.Errorf("error fetching URL: %v", err)
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("error fetching URL: received status code %d", resp.StatusCode)
+ }
+
+ content, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("error reading response body: %v", err)
+ }
+
+ return content, nil
+}
+
+// extractText extracts text content from the HTML node tree.
+func (crawler *Crawler) extractText(n *html.Node) string {
+ if n.Type == html.TextNode {
+ return n.Data
+ }
+
+ var buf bytes.Buffer
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ buf.WriteString(crawler.extractText(c))
+ }
+ return buf.String()
+}
+
+func (c *Crawler) normalizeText(text string) string {
+ text = strings.ReplaceAll(text, "ي", "ی") // Arabic Yeh to Persian Yeh
+ text = strings.ReplaceAll(text, "ك", "ک") // Arabic Kaf to Persian Kaf
+
+ // Remove punctuation using a regex.
+ reg, err := regexp.Compile("[^\\p{L}\\p{N}\\s]+")
+ if err != nil {
+ fmt.Println("Error compiling regex:", err)
+ return ""
+ }
+ normalizedText := reg.ReplaceAllString(text, " ")
+
+ return normalizedText
+}
+
+func (c *Crawler) findTopWords(text string) []string {
+ words := strings.Fields(text)
+ wordFreq := make(map[string]int)
+
+ for _, word := range words {
+ if !c.persianStopWords[word] {
+ wordFreq[word]++
+ }
+ }
+
+ type wordPair struct {
+ word string
+ count int
+ }
+ var pairs []wordPair
+ for word, count := range wordFreq {
+ pairs = append(pairs, wordPair{word, count})
+ }
+
+ sort.Slice(pairs, func(i, j int) bool {
+ return pairs[i].count > pairs[j].count
+ })
+
+ // Extract the top words.
+ topWords := []string{}
+ for i, pair := range pairs {
+ if i >= c.numOftopwords {
+ break
+ }
+ topWords = append(topWords, pair.word)
+ }
+
+ return topWords
+}
+
+func (c *Crawler) Crawl() {
+ c.crawlOnce()
+
+ ticker := time.NewTicker(1 * time.Minute)
+ defer ticker.Stop()
+
+ go func() {
+ for {
+ select {
+ case <-ticker.C:
+ c.crawlOnce()
+ }
+ }
+ }()
+}
+
+func (c *Crawler) crawlOnce() {
+ var wg sync.WaitGroup
+
+ for filePath, publisherID := range c.urls {
+ wg.Add(1)
+
+ go func(filePath string, publisherID int) {
+ defer wg.Done()
+
+ content, err := c.readHTML(filePath)
+ if err != nil {
+ log.Printf("%d+%s: Failed to read file: %v", publisherID, filePath, err)
+ }
+
+ node, err := html.Parse(bytes.NewReader(content))
+ if err != nil {
+ log.Printf("%d+%s: Failed to parse HTML: %v", publisherID, filePath, err)
+ }
+
+ rawText := c.extractText(node)
+ normalizedText := c.normalizeText(rawText)
+
+ topWords := c.findTopWords(normalizedText)
+ publisherIDStr := strconv.Itoa(int(publisherID))
+
+ c.kvstorage.Set(publisherIDStr, strings.Join(topWords, ","))
+ log.Printf("%d+%s: %s", publisherID, filePath, strings.Join(topWords, ", "))
+ }(filePath, publisherID)
+ }
+
+ go func() {
+ wg.Wait()
+ }()
+}
diff --git a/adserver/handlers/handlers.go b/adserver/handlers/handlers.go
index c03b31b..faffb51 100644
--- a/adserver/handlers/handlers.go
+++ b/adserver/handlers/handlers.go
@@ -34,7 +34,9 @@ func (h *AdServerHandler) GetAd(c *gin.Context) {
timer := prometheus.NewTimer(grafana.AdRequestDuration)
defer timer.ObserveDuration()
- chosenAd, err := h.logicService.GetBestAd()
+ publisherId, _ := strconv.Atoi(c.Param("publisherId"))
+
+ chosenAd, err := h.logicService.GetBestAd(uint(publisherId))
if err != nil {
c.JSON(http.StatusNotFound, gin.H{})
return
@@ -47,8 +49,6 @@ func (h *AdServerHandler) GetAd(c *gin.Context) {
clickReqPath := os.Getenv("CLICK_REQ_PATH")
impressionReqPath := os.Getenv("IMPRESSION_REQ_PATH")
- publisherId, _ := strconv.Atoi(c.Param("publisherId"))
-
privateKey := os.Getenv("PRIVATE_KEY")
key, _ := base64.StdEncoding.DecodeString(privateKey)
diff --git a/adserver/handlers/handlers_test.go b/adserver/handlers/handlers_test.go
index 35becc9..c5cff1d 100644
--- a/adserver/handlers/handlers_test.go
+++ b/adserver/handlers/handlers_test.go
@@ -24,7 +24,7 @@ type MockLogicService struct {
Err error
}
-func (m *MockLogicService) GetBestAd() (*dto.AdDTO, error) {
+func (m *MockLogicService) GetBestAd(publisherId uint) (*dto.AdDTO, error) {
return m.BestAd, m.Err
}
@@ -108,7 +108,7 @@ type MockLogicService_Brake struct {
BrakeAdDuration time.Duration
}
-func (m *MockLogicService_Brake) GetBestAd() (*dto.AdDTO, error) {
+func (m *MockLogicService_Brake) GetBestAd(publisherId uint) (*dto.AdDTO, error) {
return nil, nil
}
diff --git a/adserver/kvstorage/kvstorage.go b/adserver/kvstorage/kvstorage.go
new file mode 100644
index 0000000..64dcc4c
--- /dev/null
+++ b/adserver/kvstorage/kvstorage.go
@@ -0,0 +1,52 @@
+package kvstorage
+
+import (
+ "context"
+ "fmt"
+ "log"
+
+ "github.com/redis/go-redis/v9"
+)
+
+var ctx = context.Background()
+
+type KVStorageInterface interface {
+ Get(key string) (string, error)
+ Set(key string, value string) error
+}
+
+type KVStorage struct {
+ client *redis.Client
+}
+
+// NewKVStorage creates a new KVStorage instance with a connected Redis client
+func NewKVStorage(addr string) *KVStorage {
+ client := redis.NewClient(&redis.Options{
+ Addr: addr,
+ Password: "", // no password set
+ DB: 1, // use default DB
+ })
+
+ // Ping to check if connection is successful
+ _, err := client.Ping(ctx).Result()
+ if err != nil {
+ log.Printf("Failed to connect to Redis at %s for key-value storage: %v", addr, err)
+ }
+
+ return &KVStorage{client: client}
+}
+
+func (kvs *KVStorage) Get(key string) (string, error) {
+ val, err := kvs.client.Get(ctx, key).Result()
+ if err == redis.Nil {
+ return "", fmt.Errorf("could not find key %s", key) // Key does not exist
+ } else if err != nil {
+ return "", err
+ }
+ return val, nil
+}
+
+func (kvs *KVStorage) Set(key string, value string) error {
+ err := kvs.client.Set(ctx, key, value, 0).Err()
+ return err
+}
diff --git a/adserver/logic/logic.go b/adserver/logic/logic.go
index b384858..2994307 100644
--- a/adserver/logic/logic.go
+++ b/adserver/logic/logic.go
@@ -9,15 +9,17 @@ import (
"net/http"
"os"
"strconv"
+ "strings"
"time"
"YellowBloomKnapsack/mini-yektanet/adserver/grafana"
+ "YellowBloomKnapsack/mini-yektanet/adserver/kvstorage"
"YellowBloomKnapsack/mini-yektanet/common/cache"
"YellowBloomKnapsack/mini-yektanet/common/dto"
)
type LogicInterface interface {
- GetBestAd() (*dto.AdDTO, error)
+ GetBestAd(publisherId uint) (*dto.AdDTO, error)
StartTicker()
BrakeAd(adId uint)
}
@@ -26,18 +28,20 @@ type LogicService struct {
visitedAds []*dto.AdDTO
unvisitedAds []*dto.AdDTO
brakedAdsCache cache.CacheInterface
+ kvStorage kvstorage.KVStorageInterface
getAdsAPIPath string
interval int
firstChanceMaxImpressions int
}
-func NewLogicService(cache cache.CacheInterface) LogicInterface {
+func NewLogicService(cache cache.CacheInterface, kvStorage kvstorage.KVStorageInterface) LogicInterface {
interval, _ := strconv.Atoi(os.Getenv("ADS_FETCH_INTERVAL_SECS"))
firstChanceMaxImpressions, _ := strconv.Atoi(os.Getenv("FIRST_CHANCE_MAX_IMPRESSIONS"))
return &LogicService{
visitedAds: make([]*dto.AdDTO, 0),
unvisitedAds: make([]*dto.AdDTO, 0),
brakedAdsCache: cache,
+ kvStorage: kvStorage,
getAdsAPIPath: "http://" + os.Getenv("PANEL_HOSTNAME") + ":" + os.Getenv("PANEL_PORT") + os.Getenv("GET_ADS_API"),
interval: interval,
firstChanceMaxImpressions: firstChanceMaxImpressions,
@@ -71,14 +75,46 @@ func (ls *LogicService) randomOn(ads []*dto.AdDTO) *dto.AdDTO {
return ads[rand.IntN(len(ads))]
}
-func (ls *LogicService) isValid(ad *dto.AdDTO) bool {
- return !ls.brakedAdsCache.IsPresent(strconv.FormatUint(uint64(ad.ID), 10))
+func (ls *LogicService) hasIntersection(lhs, rhs []string) bool {
+ // Create a map to store elements of lhs
+ elements := make(map[string]struct{})
+ for _, item := range lhs {
+ elements[item] = struct{}{}
+ }
+
+ // Check if any element in rhs exists in the map
+ for _, item := range rhs {
+ if _, exists := elements[item]; exists {
+ return true
+ }
+ }
+
+ return false
+}
+
+func (ls *LogicService) isValid(ad *dto.AdDTO, publisherId uint) bool {
+ isBraked := ls.brakedAdsCache.IsPresent(strconv.FormatUint(uint64(ad.ID), 10))
+ if isBraked {
+ return false
+ }
+
+ if len(ad.Keywords) == 0 {
+ return true
+ }
+
+ publisherIdStr := strconv.FormatUint(uint64(publisherId), 10)
+ publisherKeywords, err := ls.kvStorage.Get(publisherIdStr)
+ if err != nil {
+ return true
+ }
+
+ return ls.hasIntersection(ad.Keywords, strings.Split(publisherKeywords, ","))
}
-func (ls *LogicService) validsOn(ads []*dto.AdDTO) []*dto.AdDTO {
+func (ls *LogicService) validsOn(ads []*dto.AdDTO, publisherId uint) []*dto.AdDTO {
result := make([]*dto.AdDTO, 0)
for _, ad := range ads {
- if ls.isValid(ad) {
+ if ls.isValid(ad, publisherId) {
result = append(result, ad)
}
}
@@ -86,9 +122,9 @@ func (ls *LogicService) validsOn(ads []*dto.AdDTO) []*dto.AdDTO {
return result
}
-func (ls *LogicService) GetBestAd() (*dto.AdDTO, error) {
- validVisitedsAds := ls.validsOn(ls.visitedAds)
- validUnvisitedsAds := ls.validsOn(ls.unvisitedAds)
+func (ls *LogicService) GetBestAd(publisherId uint) (*dto.AdDTO, error) {
+ validVisitedsAds := ls.validsOn(ls.visitedAds, publisherId)
+ validUnvisitedsAds := ls.validsOn(ls.unvisitedAds, publisherId)
if len(validUnvisitedsAds) == 0 && len(validVisitedsAds) == 0 {
log.Println("No ad was found")
@@ -172,7 +208,7 @@ func (ls *LogicService) updateAdsList() error {
}
func (ls *LogicService) StartTicker() {
- log.Println("Starting ticker...")
+ log.Println("Starting ads fetcher ticker...")
ls.updateAdsList()
go func() {
ticker := time.NewTicker(time.Duration(ls.interval) * time.Second)
diff --git a/adserver/logic/logic_test.go b/adserver/logic/logic_test.go
index 2728a48..2da41fe 100644
--- a/adserver/logic/logic_test.go
+++ b/adserver/logic/logic_test.go
@@ -14,6 +14,20 @@ import (
"github.com/stretchr/testify/assert"
)
+// Mock KVStorage
+type MockKVStorage struct {
+ kv map[string]string
+}
+
+func (s *MockKVStorage) Get(key string) (string, error) {
+ return s.kv[key], nil
+}
+
+func (s *MockKVStorage) Set(key, value string) error {
+ s.kv[key] = value
+ return nil
+}
+
// Mock CacheService
type MockCacheService struct {
mark map[string]interface{}
@@ -45,7 +59,8 @@ func TestBestScoreOn(t *testing.T) {
setupEnv()
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ads := []*dto.AdDTO{
{ID: 1, Score: 5},
@@ -66,7 +81,8 @@ func TestRandomOn(t *testing.T) {
setupEnv()
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ads := []*dto.AdDTO{
{ID: 1},
@@ -87,25 +103,27 @@ func TestValidsOn(t *testing.T) {
setupEnv()
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.brakedAdsCache.Add("2")
ls.brakedAdsCache.Add("5")
ls.brakedAdsCache.Add("6")
+ kv.Set("1", "kw2")
+
ads := []*dto.AdDTO{
- {ID: 1},
- {ID: 2},
- {ID: 3},
- {ID: 4},
- {ID: 5},
- {ID: 6},
+ {ID: 1, Keywords: []string{"kw1", "kw2"}},
+ {ID: 2, Keywords: []string{}},
+ {ID: 3, Keywords: []string{"kw2", "kw3"}},
+ {ID: 4, Keywords: []string{"kw4", "kw5"}},
+ {ID: 5, Keywords: []string{}},
+ {ID: 6, Keywords: []string{}},
}
- validAds := ls.validsOn(ads)
- assert.Equal(t, 3, len(validAds))
+ validAds := ls.validsOn(ads, uint(1))
+ assert.Equal(t, 2, len(validAds))
assert.Equal(t, uint(1), validAds[0].ID)
assert.Equal(t, uint(3), validAds[1].ID)
- assert.Equal(t, uint(4), validAds[2].ID)
}
func TestUpdateAdsList(t *testing.T) {
@@ -123,7 +141,8 @@ func TestUpdateAdsList(t *testing.T) {
defer ts.Close()
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.getAdsAPIPath = ts.URL
err := ls.updateAdsList()
@@ -136,7 +155,8 @@ func TestStartTicker(t *testing.T) {
setupEnv()
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.interval = 1
ads := []dto.AdDTO{
@@ -168,8 +188,8 @@ func TestBrakeAd(t *testing.T) {
duration := 200 * time.Millisecond
cache := &MockCacheService{make(map[string]interface{})}
- service := NewLogicService(cache)
- ls, _ := service.(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.BrakeAd(adId)
@@ -191,9 +211,10 @@ func TestBrakeAd(t *testing.T) {
func TestGetBestAd_NoAdsAvailable(t *testing.T) {
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
- _, err := ls.GetBestAd()
+ _, err := ls.GetBestAd(uint(1))
assert.Error(t, err)
assert.Equal(t, "no ad was found", err.Error())
}
@@ -202,26 +223,28 @@ func TestGetBestAd_OnlyUnvisitedAdsAvailable(t *testing.T) {
os.Setenv("UNVISITED_CHANCE", "100")
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.unvisitedAds = []*dto.AdDTO{
{ID: 1, Score: 5},
{ID: 2, Score: 10},
}
- bestAd, err := ls.GetBestAd()
+ bestAd, err := ls.GetBestAd(uint(1))
assert.NoError(t, err)
assert.Contains(t, []uint{1, 2}, bestAd.ID)
}
func TestGetBestAd_OnlyVisitedAdsAvailable(t *testing.T) {
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.visitedAds = []*dto.AdDTO{
{ID: 1, Score: 5},
{ID: 2, Score: 10},
}
- bestAd, err := ls.GetBestAd()
+ bestAd, err := ls.GetBestAd(uint(1))
assert.NoError(t, err)
assert.Equal(t, uint(2), bestAd.ID)
}
@@ -230,7 +253,8 @@ func TestGetBestAd_BothAdsAvailable_UnvisitedChance100(t *testing.T) {
os.Setenv("UNVISITED_CHANCE", "100")
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.visitedAds = []*dto.AdDTO{
{ID: 1, Score: 5},
}
@@ -238,7 +262,7 @@ func TestGetBestAd_BothAdsAvailable_UnvisitedChance100(t *testing.T) {
{ID: 2, Score: 10},
}
- bestAd, err := ls.GetBestAd()
+ bestAd, err := ls.GetBestAd(uint(1))
assert.NoError(t, err)
assert.Equal(t, uint(2), bestAd.ID)
}
@@ -247,7 +271,8 @@ func TestGetBestAd_BothAdsAvailable_UnvisitedChance0(t *testing.T) {
os.Setenv("UNVISITED_CHANCE", "0")
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.visitedAds = []*dto.AdDTO{
{ID: 1, Score: 5},
}
@@ -255,7 +280,7 @@ func TestGetBestAd_BothAdsAvailable_UnvisitedChance0(t *testing.T) {
{ID: 2, Score: 10},
}
- bestAd, err := ls.GetBestAd()
+ bestAd, err := ls.GetBestAd(uint(1))
assert.NoError(t, err)
assert.Equal(t, uint(1), bestAd.ID)
}
@@ -264,7 +289,8 @@ func TestGetBestAd_ValidUnvisitedAdsAvailable(t *testing.T) {
os.Setenv("UNVISITED_CHANCE", "100")
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.unvisitedAds = []*dto.AdDTO{
{ID: 1, Score: 5},
{ID: 2, Score: 10},
@@ -273,14 +299,15 @@ func TestGetBestAd_ValidUnvisitedAdsAvailable(t *testing.T) {
// Braking one of the ads to make it invalid
ls.BrakeAd(1)
- bestAd, err := ls.GetBestAd()
+ bestAd, err := ls.GetBestAd(uint(1))
assert.NoError(t, err)
assert.Equal(t, uint(2), bestAd.ID)
}
func TestGetBestAd_ValidVisitedAdsAvailable(t *testing.T) {
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.visitedAds = []*dto.AdDTO{
{ID: 1, Score: 5},
{ID: 2, Score: 10},
@@ -289,7 +316,7 @@ func TestGetBestAd_ValidVisitedAdsAvailable(t *testing.T) {
// Braking one of the ads to make it invalid
ls.BrakeAd(2)
- bestAd, err := ls.GetBestAd()
+ bestAd, err := ls.GetBestAd(uint(1))
assert.NoError(t, err)
assert.Equal(t, uint(1), bestAd.ID)
}
@@ -298,7 +325,8 @@ func TestGetBestAd_ValidUnvisitedAndVisitedAdsAvailable(t *testing.T) {
os.Setenv("UNVISITED_CHANCE", "50")
cache := &MockCacheService{make(map[string]interface{})}
- ls := NewLogicService(cache).(*LogicService)
+ kv := &MockKVStorage{make(map[string]string)}
+ ls := NewLogicService(cache, kv).(*LogicService)
ls.visitedAds = []*dto.AdDTO{
{ID: 1, Score: 5},
}
@@ -309,7 +337,33 @@ func TestGetBestAd_ValidUnvisitedAndVisitedAdsAvailable(t *testing.T) {
// Braking the unvisited ad to make it invalid
ls.BrakeAd(2)
- bestAd, err := ls.GetBestAd()
+ bestAd, err := ls.GetBestAd(uint(1))
assert.NoError(t, err)
assert.Equal(t, uint(1), bestAd.ID)
}
+
+func TestHasIntersection(t *testing.T) {
+ ls := &LogicService{}
+
+ tests := []struct {
+ lhs []string
+ rhs []string
+ expected bool
+ }{
+ {[]string{"apple", "banana", "cherry"}, []string{"grape", "banana", "kiwi"}, true},
+ {[]string{"apple", "banana", "cherry"}, []string{"grape", "kiwi"}, false},
+ {[]string{"apple", "banana", "cherry"}, []string{"apple", "kiwi"}, true},
+ {[]string{}, []string{"grape", "kiwi"}, false},
+ {[]string{"apple", "banana"}, []string{}, false},
+ {[]string{}, []string{}, false},
+ {[]string{"apple", "banana", "banana"}, []string{"banana"}, true},
+ {[]string{"apple", "banana", "cherry"}, []string{"cherry", "banana"}, true},
+ }
+
+ for _, test := range tests {
+ result := ls.hasIntersection(test.lhs, test.rhs)
+ if result != test.expected {
+ t.Errorf("hasIntersection(%v, %v) = %v; expected %v", test.lhs, test.rhs, result, test.expected)
+ }
+ }
+}
diff --git a/adserver/main.go b/adserver/main.go
index 0db8953..d8f1f00 100644
--- a/adserver/main.go
+++ b/adserver/main.go
@@ -11,22 +11,27 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"YellowBloomKnapsack/mini-yektanet/adserver/cache"
+ "YellowBloomKnapsack/mini-yektanet/adserver/crawler"
"YellowBloomKnapsack/mini-yektanet/adserver/handlers"
+ "YellowBloomKnapsack/mini-yektanet/adserver/kvstorage"
"YellowBloomKnapsack/mini-yektanet/adserver/logic"
"YellowBloomKnapsack/mini-yektanet/common/tokenhandler"
)
func main() {
- if err := godotenv.Load(".env", "../common/.env", "../eventserver/.env", "../panel/.env"); err != nil {
+ if err := godotenv.Load(".env", "../common/.env", "../eventserver/.env", "../panel/.env", "../publisherwebsite/.env"); err != nil {
log.Fatal("Error loading .env file")
}
tokenHandler := tokenhandler.NewTokenHandlerService()
redisUrl := os.Getenv("REDIS_HOST") + ":" + os.Getenv("REDIS_PORT")
cacheService := cache.NewAdServerCache(redisUrl)
- logicService := logic.NewLogicService(cacheService)
+ kvStorageService := kvstorage.NewKVStorage(redisUrl)
+ crawlerService := crawler.NewCrawler(kvStorageService)
+ logicService := logic.NewLogicService(cacheService, kvStorageService)
handler := handlers.NewAdServerHandler(logicService, tokenHandler)
+ crawlerService.Crawl()
r := gin.Default()
diff --git a/common/dto/AdDto.go b/common/dto/AdDto.go
index 09d64ba..2932c37 100644
--- a/common/dto/AdDto.go
+++ b/common/dto/AdDto.go
@@ -10,4 +10,5 @@ type AdDTO struct {
Impressions int
BalanceAdvertiser int64
Score float64
+ Keywords []string
}
diff --git a/common/dto/Events.pb.go b/common/dto/Events.pb.go
index c483a1b..b85f21b 100644
--- a/common/dto/Events.pb.go
+++ b/common/dto/Events.pb.go
@@ -16,7 +16,6 @@ import (
const (
// Verify that this generated code is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
- // Verify that runtime/protoimpl is sufficiently up-to-date.
_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
)
diff --git a/common/models/ad.go b/common/models/ad.go
index 74d8567..dc5b0a9 100644
--- a/common/models/ad.go
+++ b/common/models/ad.go
@@ -13,4 +13,5 @@ type Ad struct {
AdvertiserID uint
Advertiser Advertiser
AdsInteractions []AdsInteraction
+ Keywords Keyword
}
diff --git a/common/models/keyword.go b/common/models/keyword.go
new file mode 100644
index 0000000..143f93f
--- /dev/null
+++ b/common/models/keyword.go
@@ -0,0 +1,9 @@
+package models
+
+import "gorm.io/gorm"
+
+type Keyword struct {
+ gorm.Model
+ AdID uint `gorm:"index"`
+ Keywords string `gorm:"type:varchar(255)"`
+}
diff --git a/panel/database/database.go b/panel/database/database.go
index 511f65c..69bcb1c 100644
--- a/panel/database/database.go
+++ b/panel/database/database.go
@@ -56,6 +56,7 @@ func InitDB() {
DB.AutoMigrate(&models.AdsInteraction{})
DB.AutoMigrate(&models.Advertiser{})
DB.AutoMigrate(&models.Transaction{})
+ DB.AutoMigrate(&models.Keyword{})
err = initPublishers()
if err != nil {
log.Fatal(err)
@@ -90,6 +91,7 @@ func InitTestDB() {
DB.AutoMigrate(&models.AdsInteraction{})
DB.AutoMigrate(&models.Advertiser{})
DB.AutoMigrate(&models.Transaction{})
+ DB.AutoMigrate(&models.Keyword{})
if err != nil {
log.Fatal(err)
}
diff --git a/panel/handlers/adhandler.go b/panel/handlers/adhandler.go
index 52f6c65..486e732 100644
--- a/panel/handlers/adhandler.go
+++ b/panel/handlers/adhandler.go
@@ -1,14 +1,13 @@
package handlers
import (
- "net/http"
- "os"
- "strconv"
-
"YellowBloomKnapsack/mini-yektanet/common/dto"
"YellowBloomKnapsack/mini-yektanet/common/models"
"YellowBloomKnapsack/mini-yektanet/panel/database"
"YellowBloomKnapsack/mini-yektanet/panel/logic"
+ "net/http"
+ "os"
+ "strconv"
"github.com/gin-gonic/gin"
)
@@ -17,6 +16,7 @@ func GetActiveAds(c *gin.Context) {
var ads []models.Ad
result := database.DB.Preload("Advertiser").
Preload("AdsInteractions").
+ Preload("Keywords").
Joins("JOIN advertisers ON advertisers.id = ads.advertiser_id").
Where("ads.active = ? AND advertisers.balance > ?", true, 0).
Find(&ads)
@@ -28,6 +28,7 @@ func GetActiveAds(c *gin.Context) {
var adDTOs []dto.AdDTO
for _, ad := range ads {
impressionsCount := getImpressionCounts(ad.AdsInteractions)
+
adDTO := dto.AdDTO{
ID: ad.ID,
Text: ad.Text,
@@ -38,6 +39,7 @@ func GetActiveAds(c *gin.Context) {
BalanceAdvertiser: ad.Advertiser.Balance,
Impressions: impressionsCount,
Score: logic.GetScore(ad, impressionsCount),
+ Keywords: logic.SplitAndClean(ad.Keywords.Keywords),
}
adDTOs = append(adDTOs, adDTO)
}
diff --git a/panel/handlers/advertiserhandler.go b/panel/handlers/advertiserhandler.go
index 68a7a99..7b33184 100644
--- a/panel/handlers/advertiserhandler.go
+++ b/panel/handlers/advertiserhandler.go
@@ -6,6 +6,7 @@ import (
"os"
"path"
"strconv"
+ "strings"
"time"
"YellowBloomKnapsack/mini-yektanet/common/models"
@@ -22,14 +23,14 @@ const (
INTBASE = 10
INTBIT32 = 32
INTBIT64 = 64
- itemsPerPage = 4
+ itemsPerPage = 47
)
func AdvertiserPanel(c *gin.Context) {
advertiserUserName := c.Param("username")
var advertiser models.Advertiser
- result := database.DB.Preload("Ads").Where("username = ?", advertiserUserName).First(&advertiser)
+ result := database.DB.Preload("Ads").Preload("Ads.Keywords").Where("username = ?", advertiserUserName).First(&advertiser)
if result.Error != nil {
if result.Error == gorm.ErrRecordNotFound {
fmt.Printf("No advertiser found with username %s, creating a new one.\n", advertiserUserName)
@@ -149,13 +150,14 @@ func AddFunds(c *gin.Context) {
}
func CreateAd(c *gin.Context) {
- // TODO: Get advertiser ID from session
advertiserUserName := c.Param("username")
var advertiser models.Advertiser
result := database.DB.Where("username = ?", advertiserUserName).First(&advertiser)
if result.Error != nil {
c.AbortWithStatus(http.StatusInternalServerError)
+ return
}
+
title := c.PostForm("title")
website := c.PostForm("website")
bid, _ := strconv.ParseInt(c.PostForm("bid"), INTBASE, INTBIT64)
@@ -164,23 +166,16 @@ func CreateAd(c *gin.Context) {
return
}
- // Handle file upload
file, _ := c.FormFile("image")
-
- // Create a unique filename
filename := fmt.Sprintf("%d_%s", time.Now().UnixNano(), file.Filename)
-
- // Define the path where the image will be saved
uploadDir := "static/uploads/"
filepath := path.Join(uploadDir, filename)
- // Ensure the upload directory exists
if err := os.MkdirAll(uploadDir, os.ModePerm); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create upload directory"})
return
}
- // Save the file
if err := c.SaveUploadedFile(file, filepath); err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to save file"})
return
@@ -188,7 +183,7 @@ func CreateAd(c *gin.Context) {
ad := models.Ad{
Text: title,
- ImagePath: "/" + filepath, // Store the path relative to the server root
+ ImagePath: "/" + filepath,
Bid: bid,
AdvertiserID: advertiser.ID,
Website: website,
@@ -196,6 +191,22 @@ func CreateAd(c *gin.Context) {
database.DB.Create(&ad)
+ // Extract and save keywords
+ keywords := []string{
+ c.PostForm("keyword1"),
+ c.PostForm("keyword2"),
+ c.PostForm("keyword3"),
+ c.PostForm("keyword4"),
+ }
+
+ keywordString := strings.Join(keywords, ",")
+
+ keywordRecord := models.Keyword{
+ AdID: ad.ID,
+ Keywords: keywordString,
+ }
+ database.DB.Create(&keywordRecord)
+
c.Redirect(http.StatusSeeOther, fmt.Sprintf("/advertiser/%s/panel", advertiserUserName))
// average bid is calculated based on number of bids
@@ -238,7 +249,7 @@ func AdReport(c *gin.Context) {
adID, _ := strconv.ParseUint(c.Param("id"), INTBASE, INTBIT32)
var ad models.Ad
- database.DB.Preload("AdsInteractions").Preload("Advertiser").First(&ad, adID)
+ database.DB.Preload("AdsInteractions").Preload("Keywords").Preload("Advertiser").First(&ad, adID)
impressions := 0
clicks := 0
@@ -262,8 +273,10 @@ func AdReport(c *gin.Context) {
"CTR": ctr,
"TotalCost": ad.TotalCost,
"Website": ad.Website,
+ "Keywords": ad.Keywords,
})
}
+
func HandleEditAd(c *gin.Context) {
username := c.Param("username")
adID, err := strconv.Atoi(c.PostForm("ad_id"))
@@ -329,6 +342,7 @@ func HandleEditAd(c *gin.Context) {
c.Redirect(http.StatusSeeOther, "/advertiser/"+username+"/panel")
}
+
func removeFileIfExists(filePath string) error {
// Check if the file exists
_, err := os.Stat(filePath)
diff --git a/panel/logic/logic.go b/panel/logic/logic.go
index aa348ef..441c667 100644
--- a/panel/logic/logic.go
+++ b/panel/logic/logic.go
@@ -4,6 +4,7 @@ import (
"database/sql"
"os"
"strconv"
+ "strings"
"time"
"YellowBloomKnapsack/mini-yektanet/common/models"
@@ -38,3 +39,24 @@ func GetSumOfBids(db *gorm.DB, adID uint) (int64, error) {
return 0, err
}
}
+
+func SplitAndClean(input string) []string {
+ // Split the input string by commas
+ parts := strings.Split(input, ",")
+
+ // Create a slice to hold the cleaned parts
+ var result []string
+
+ // Loop through the parts
+ for _, part := range parts {
+ // Trim any leading or trailing whitespace
+ trimmed := strings.TrimSpace(part)
+
+ // Add the non-empty trimmed part to the result
+ if trimmed != "" {
+ result = append(result, trimmed)
+ }
+ }
+
+ return result
+}
diff --git a/panel/templates/ad_report.html b/panel/templates/ad_report.html
index 1f08af1..76794e1 100644
--- a/panel/templates/ad_report.html
+++ b/panel/templates/ad_report.html
@@ -22,6 +22,7 @@ Ad Report
Clicks: {{ .Clicks }}
CTR: {{ printf "%.2f%%" .CTR }}
Total Cost: {{ .TotalCost }}
+ Keywords: {{ .Keywords.Keywords }}
Website: {{.Website}}
Back to Panel
diff --git a/panel/templates/advertiser_panel.html b/panel/templates/advertiser_panel.html
index 51d77fc..ea416fe 100644
--- a/panel/templates/advertiser_panel.html
+++ b/panel/templates/advertiser_panel.html
@@ -9,74 +9,42 @@
-
-
Advertiser Panel
-
-
Account Balance: ${{ .Balance }}
-
-
-
-
Create New Ad
-
-
-
-
Transactions
-
-
-
- | ID |
- Description |
- Amount |
- Successful |
- Income |
- Time |
-
-
-
- {{ range .Transactions }}
-
- | {{ .ID }} |
- {{ .Description }} |
- {{ .Amount }} |
- {{ if .Successful }}Yes{{ else }}No{{ end }} |
- {{ if .Income }}Yes{{ else }}No{{ end }} |
- {{ .Time.Format "2006-01-02 15:04:05" }} |
-
- {{ end }}
-
-
+
+
Advertiser Panel
+
+
Account Balance: ${{ .Balance }}
+
+
+
+
Create New Ad
+
+
+
+
+
-
-
Your Ads
- {{ range .Ads }}
+
+
+
+
+
+
Your Ads
+ {{ range .Ads }}
{{ .Text }}
Bid: ${{ .Bid }}
Website: {{ .Website }}
Status: {{ if .Active }}Active{{ else }}Inactive{{ end }}
+
Keywords: {{ .Keywords.Keywords }}
+ {{ end }}
+
+
+
Transactions
+
+
+
+ | ID |
+ Description |
+ Amount |
+ Successful |
+ Income |
+ Time |
+
+
+
+ {{ range .Transactions }}
+
+ | {{ .ID }} |
+ {{ .Description }} |
+ {{ .Amount }} |
+ {{ if .Successful }}Yes{{ else }}No{{ end }} |
+ {{ if .Income }}Yes{{ else }}No{{ end }} |
+ {{ .Time.Format "2006-01-02 15:04:05" }} |
+
+ {{ end }}
+
+
+
+
-
-
+
+
-