Merge pull request #21 from xxxsen/xxxsen/feature/add_rule_api

Xxxsen/feature/add rule api

capture/capture.go

@@ -61,10 +61,24 @@ func (c *Capture) resolveFileInfo(fc *model.FileContext, file string) error {
 	fc.FileName = filepath.Base(file)
 	fc.FileExt = filepath.Ext(file)
 	fileNoExt := fc.FileName[:len(fc.FileName)-len(fc.FileExt)]
+	//番号改写
+	fileNoExt, err := c.c.NumberRewriter.Rewrite(fileNoExt)
+	if err != nil {
+		return fmt.Errorf("rewrite number before parse failed, err:%w", err)
+	}
+	//番号解析
 	info, err := number.Parse(fileNoExt)
 	if err != nil {
 		return fmt.Errorf("parse number failed, err:%w", err)
 	}
+	//规则测试
+	//是否无码
+	ok, _ := c.c.UncensorTester.Test(info.GetNumberID())
+	info.SetExternalFieldUncensor(ok)
+	//尝试分类
+	cat, _, _ := c.c.NumberCategorier.Match(info.GetNumberID())
+	info.SetExternalFieldCategory(cat)
+
 	fc.Number = info
 	fc.SaveFileBase = fc.Number.GenerateFileName()
 	return nil

capture/config.go

@@ -1,6 +1,8 @@
 package capture
 
 import (
+	"fmt"
+	"yamdc/capture/ruleapi"
 	"yamdc/processor"
 	"yamdc/searcher"
 )
@@ -13,8 +15,8 @@ const (
 	NamingNumber       = "NUMBER"
 )
 
-const (
-	defaultNamingRule = NamingReleaseYear + "/" + NamingActor + "/" + NamingNumber
+var (
+	defaultNamingRule = fmt.Sprintf("{%s}/{%s}", NamingReleaseYear, NamingNumber)
 )
 
 type config struct {
@@ -24,6 +26,9 @@ type config struct {
 	SaveDir           string
 	Naming            string
 	ExtraMediaExtList []string
+	UncensorTester    ruleapi.ITester
+	NumberRewriter    ruleapi.IRewriter
+	NumberCategorier  ruleapi.IMatcher
 }
 
 type Option func(c *config)
@@ -63,3 +68,21 @@ func WithExtraMediaExtList(lst []string) Option {
 		c.ExtraMediaExtList = lst
 	}
 }
+
+func WithUncensorTester(t ruleapi.ITester) Option {
+	return func(c *config) {
+		c.UncensorTester = t
+	}
+}
+
+func WithNumberRewriter(t ruleapi.IRewriter) Option {
+	return func(c *config) {
+		c.NumberRewriter = t
+	}
+}
+
+func WithNumberCategorier(t ruleapi.IMatcher) Option {
+	return func(c *config) {
+		c.NumberCategorier = t
+	}
+}

capture/ruleapi/matcher.go

@@ -0,0 +1,5 @@
+package ruleapi
+
+type IMatcher interface {
+	Match(res string) (string, bool, error)
+}

capture/ruleapi/regexp_matcher.go

@@ -0,0 +1,46 @@
+package ruleapi
+
+type RegexpMatchRule struct {
+	Regexp []string
+	Match  string
+}
+
+type regexpMatchItem struct {
+	reg   ITester
+	match string
+}
+
+type RegexpMatcher struct {
+	matchList []*regexpMatchItem
+}
+
+func NewRegexpMatcher() *RegexpMatcher {
+	return &RegexpMatcher{}
+}
+
+func (m *RegexpMatcher) AddRules(rules ...RegexpMatchRule) error {
+	for _, rule := range rules {
+		t := NewRegexpTester()
+		if err := t.AddRules(rule.Regexp...); err != nil {
+			return err
+		}
+		m.matchList = append(m.matchList, &regexpMatchItem{
+			reg:   t,
+			match: rule.Match,
+		})
+	}
+	return nil
+}
+
+func (m *RegexpMatcher) Match(res string) (string, bool, error) {
+	for _, matcher := range m.matchList {
+		ok, err := matcher.reg.Test(res)
+		if err != nil {
+			return "", false, err
+		}
+		if ok {
+			return matcher.match, true, nil
+		}
+	}
+	return "", false, nil
+}

capture/ruleapi/regexp_rewriter.go

@@ -0,0 +1,43 @@
+package ruleapi
+
+import (
+	"regexp"
+)
+
+type RegexpRewriteRule struct {
+	Rule    string
+	Rewrite string
+}
+
+type regexpRewriteItem struct {
+	reg     *regexp.Regexp
+	rewrite string
+}
+
+type RegexpRewriter struct {
+	rewriteList []*regexpRewriteItem
+}
+
+func NewRegexpRewriter() *RegexpRewriter {
+	return &RegexpRewriter{}
+}
+
+func (r *RegexpRewriter) AddRules(rs ...RegexpRewriteRule) error {
+	for _, rule := range rs {
+		reg, err := regexp.Compile(rule.Rule)
+		if err != nil {
+			return err
+		}
+		r.rewriteList = append(r.rewriteList, &regexpRewriteItem{
+			reg:     reg,
+			rewrite: rule.Rewrite,
+		})
+	}
+	return nil
+}
+func (r *RegexpRewriter) Rewrite(res string) (string, error) {
+	for _, rewriter := range r.rewriteList {
+		res = rewriter.reg.ReplaceAllString(res, rewriter.rewrite)
+	}
+	return res, nil
+}

capture/ruleapi/regexp_tester.go

@@ -0,0 +1,33 @@
+package ruleapi
+
+import (
+	"regexp"
+)
+
+type RegexpTester struct {
+	testList []*regexp.Regexp
+}
+
+func NewRegexpTester() *RegexpTester {
+	return &RegexpTester{}
+}
+
+func (t *RegexpTester) AddRules(rules ...string) error {
+	for _, rule := range rules {
+		reg, err := regexp.Compile(rule)
+		if err != nil {
+			return err
+		}
+		t.testList = append(t.testList, reg)
+	}
+	return nil
+}
+
+func (t *RegexpTester) Test(res string) (bool, error) {
+	for _, tester := range t.testList {
+		if tester.MatchString(res) {
+			return true, nil
+		}
+	}
+	return false, nil
+}

capture/ruleapi/rewriter.go

@@ -0,0 +1,5 @@
+package ruleapi
+
+type IRewriter interface {
+	Rewrite(res string) (string, error)
+}

capture/ruleapi/tester.go

@@ -0,0 +1,5 @@
+package ruleapi
+
+type ITester interface {
+	Test(res string) (bool, error)
+}

config/config.go

@@ -29,62 +29,50 @@ type NetworkConfig struct {
 	Proxy   string `json:"proxy"`
 }
 
+type NumberRewriteRule struct {
+	Remark  string `json:"remark"`
+	Rule    string `json:"rule"`
+	Rewrite string `json:"rewrite"`
+}
+
+type NumberCategoryRule struct {
+	Remark   string   `json:"remark"`
+	Rules    []string `json:"rules"`
+	Category string   `json:"category"`
+}
+
+type NumberRule struct {
+	NumberUncensorRules []string             `json:"number_uncensor_rules"`
+	NumberRewriteRules  []NumberRewriteRule  `json:"number_rewrite_rules"`
+	NumberCategoryRule  []NumberCategoryRule `json:"number_category_rules"`
+}
+
 type Config struct {
-	ScanDir         string                 `json:"scan_dir"`
-	SaveDir         string                 `json:"save_dir"`
-	DataDir         string                 `json:"data_dir"`
-	Naming          string                 `json:"naming"`
-	PluginConfig    map[string]interface{} `json:"plugin_config"`
-	HandlerConfig   map[string]interface{} `json:"handler_config"`
-	Plugins         []string               `json:"plugins"`
-	CategoryPlugins []CategoryPlugin       `json:"category_plugins"`
-	Handlers        []string               `json:"handlers"`
-	ExtraMediaExts  []string               `json:"extra_media_exts"`
-	LogConfig       logger.LogConfig       `json:"log_config"`
-	Dependencies    []Dependency           `json:"dependencies"`
-	NetworkConfig   NetworkConfig          `json:"network_config"`
+	ScanDir           string                 `json:"scan_dir"`
+	SaveDir           string                 `json:"save_dir"`
+	DataDir           string                 `json:"data_dir"`
+	Naming            string                 `json:"naming"`
+	PluginConfig      map[string]interface{} `json:"plugin_config"`
+	HandlerConfig     map[string]interface{} `json:"handler_config"`
+	Plugins           []string               `json:"plugins"`
+	CategoryPlugins   []CategoryPlugin       `json:"category_plugins"`
+	Handlers          []string               `json:"handlers"`
+	ExtraMediaExts    []string               `json:"extra_media_exts"`
+	LogConfig         logger.LogConfig       `json:"log_config"`
+	Dependencies      []Dependency           `json:"dependencies"`
+	NetworkConfig     NetworkConfig          `json:"network_config"`
+	NumberDefaultRule NumberRule             `json:"number_default_rule"` //默认规则
+	NumberUserRule    NumberRule             `json:"number_user_rule"`    //用户自定义规则, 最终跟默认规则进行合并
 }
 
 func defaultConfig() *Config {
 	return &Config{
-		Plugins: []string{
-			"javbus",
-			"javhoo",
-			"airav",
-			"javdb",
-			"jav321",
-			"caribpr",
-			"18av",
-			"njav",
-			"missav",
-			"freejavbt",
-			"tktube",
-			"avsox",
-		},
-		CategoryPlugins: []CategoryPlugin{
-			//如果存在分配配置, 那么当番号被识别为特定分类的场景下, 将会使用分类插件直接查询
-			{Name: "FC2", Plugins: []string{"fc2", "18av", "njav", "freejavbt", "tktube", "avsox", "fc2ppvdb"}},
-		},
-		Handlers: []string{
-			"image_transcoder",
-			"poster_cropper",
-			"watermark_maker",
-			"actor_spliter",
-			"tag_padder",
-			"duration_fixer",
-			"number_title",
-			"translater",
-		},
-		LogConfig: logger.LogConfig{
-			Level:   "info",
-			Console: true,
-		},
-		Dependencies: []Dependency{
-			{Link: "https://github.com/Kagami/go-face-testdata/raw/master/models/shape_predictor_5_face_landmarks.dat", RelPath: "models/shape_predictor_5_face_landmarks.dat"},
-			{Link: "https://github.com/Kagami/go-face-testdata/raw/master/models/dlib_face_recognition_resnet_model_v1.dat", RelPath: "models/dlib_face_recognition_resnet_model_v1.dat"},
-			{Link: "https://github.com/Kagami/go-face-testdata/raw/master/models/mmod_human_face_detector.dat", RelPath: "models/mmod_human_face_detector.dat"},
-			{Link: "https://github.com/esimov/pigo/raw/master/cascade/facefinder", RelPath: "models/facefinder"},
-		},
+		Plugins:           sysPlugins,
+		CategoryPlugins:   sysCategoryPlugins,
+		Handlers:          sysHandler,
+		LogConfig:         sysLogConfig,
+		Dependencies:      sysDependencies,
+		NumberDefaultRule: sysNumberRule,
 	}
 }
 

config/dependency_config.go

@@ -0,0 +1,8 @@
+package config
+
+var sysDependencies = []Dependency{
+	{Link: "https://github.com/Kagami/go-face-testdata/raw/master/models/shape_predictor_5_face_landmarks.dat", RelPath: "models/shape_predictor_5_face_landmarks.dat"},
+	{Link: "https://github.com/Kagami/go-face-testdata/raw/master/models/dlib_face_recognition_resnet_model_v1.dat", RelPath: "models/dlib_face_recognition_resnet_model_v1.dat"},
+	{Link: "https://github.com/Kagami/go-face-testdata/raw/master/models/mmod_human_face_detector.dat", RelPath: "models/mmod_human_face_detector.dat"},
+	{Link: "https://github.com/esimov/pigo/raw/master/cascade/facefinder", RelPath: "models/facefinder"},
+}

config/handler_config.go

@@ -0,0 +1,12 @@
+package config
+
+var sysHandler = []string{
+	"image_transcoder",
+	"poster_cropper",
+	"watermark_maker",
+	"actor_spliter",
+	"tag_padder",
+	"duration_fixer",
+	"number_title",
+	"translater",
+}

config/log_config.go

@@ -0,0 +1,8 @@
+package config
+
+import "github.com/xxxsen/common/logger"
+
+var sysLogConfig = logger.LogConfig{
+	Level:   "info",
+	Console: true,
+}

config/plugin_config.go

@@ -0,0 +1,21 @@
+package config
+
+var sysPlugins = []string{
+	"javbus",
+	"javhoo",
+	"airav",
+	"javdb",
+	"jav321",
+	"caribpr",
+	"18av",
+	"njav",
+	"missav",
+	"freejavbt",
+	"tktube",
+	"avsox",
+}
+
+var sysCategoryPlugins = []CategoryPlugin{
+	//如果存在分配配置, 那么当番号被识别为特定分类的场景下, 将会使用分类插件直接查询
+	{Name: "FC2", Plugins: []string{"fc2", "18av", "njav", "freejavbt", "tktube", "avsox", "fc2ppvdb"}},
+}