diff --git a/capture/capture.go b/capture/capture.go index 4b46a4d..8806d05 100644 --- a/capture/capture.go +++ b/capture/capture.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "time" + "yamdc/envflag" "yamdc/model" "yamdc/nfo" "yamdc/number" @@ -320,7 +321,7 @@ func (c *Capture) saveMediaData(ctx context.Context, fc *model.FileContext) erro } func (c *Capture) moveMovie(fc *model.FileContext, src string, dst string) error { - if c.c.LinkMode { + if envflag.IsEnableLinkMode() { return c.moveMovieByLink(fc, src, dst) } return c.moveMovieDirect(fc, src, dst) diff --git a/capture/config.go b/capture/config.go index a3d49b1..37849d4 100644 --- a/capture/config.go +++ b/capture/config.go @@ -24,7 +24,6 @@ type config struct { SaveDir string Naming string ExtraMediaExtList []string - LinkMode bool } type Option func(c *config) @@ -59,12 +58,6 @@ func WithNamingRule(r string) Option { } } -func WithEnableLinkMode(v bool) Option { - return func(c *config) { - c.LinkMode = v - } -} - func WithExtraMediaExtList(lst []string) Option { return func(c *config) { c.ExtraMediaExtList = lst diff --git a/envflag/envflag.go b/envflag/envflag.go new file mode 100644 index 0000000..c611908 --- /dev/null +++ b/envflag/envflag.go @@ -0,0 +1,33 @@ +package envflag + +import ( + "github.com/kelseyhightower/envconfig" +) + +var defaultInst = &EnvFlag{} + +type EnvFlag struct { + EnableSearchMetaCache bool `envconfig:"enable_search_meta_cache" default:"true"` + EnableLinkMode bool `envconfig:"enable_link_mode"` +} + +func GetFlag() *EnvFlag { + return defaultInst +} + +func Init() error { + fg := &EnvFlag{} + if err := envconfig.Process("exec_flag", fg); err != nil { + return err + } + defaultInst = fg + return nil +} + +func IsEnableSearchMetaCache() bool { + return GetFlag().EnableSearchMetaCache +} + +func IsEnableLinkMode() bool { + return GetFlag().EnableLinkMode +} diff --git a/go.mod b/go.mod index 5e93c43..c0561f6 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/google/pprof v0.0.0-20240910150728-a0b0bb1d4134 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/kelseyhightower/envconfig v1.4.0 github.com/klauspost/compress v1.17.9 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/onsi/ginkgo/v2 v2.20.2 // indirect diff --git a/go.sum b/go.sum index 9a524ad..f68cbc1 100644 --- a/go.sum +++ b/go.sum @@ -43,6 +43,8 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/imroc/req/v3 v3.48.0 h1:IYuMGetuwLzOOTzDCquDqs912WNwpsPK0TBXWPIvoqg= github.com/imroc/req/v3 v3.48.0/go.mod h1:weam9gmyb00QnOtu6HXSnk44dNFkIUQb5QdMx13FeUU= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= diff --git a/main.go b/main.go index 6bbb92f..977060c 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "yamdc/client" "yamdc/config" "yamdc/dependency" + "yamdc/envflag" "yamdc/face" "yamdc/face/goface" "yamdc/face/pigo" @@ -26,8 +27,8 @@ import ( "github.com/xxxsen/common/logutil" "go.uber.org/zap" - "yamdc/searcher/plugin" - _ "yamdc/searcher/plugin/airav" + "yamdc/searcher/plugin/factory" + _ "yamdc/searcher/plugin/register" ) var conf = flag.String("config", "./config.json", "config file") @@ -47,6 +48,12 @@ func main() { logkit.Fatal("ensure dependencies failed", zap.Error(err)) } logkit.Info("check dependencies finish...") + + if err := envflag.Init(); err != nil { + logkit.Fatal("init envflag failed", zap.Error(err)) + } + logkit.Info("read env flags", zap.Any("flag", *envflag.GetFlag())) + store.SetStorage(store.MustNewSqliteStorage(filepath.Join(c.DataDir, "cache", "cache.db"))) if err := translator.Init(); err != nil { logkit.Error("init translater failed", zap.Error(err)) @@ -54,7 +61,7 @@ func main() { if err := initFace(filepath.Join(c.DataDir, "models")); err != nil { logkit.Error("init face recognizer failed", zap.Error(err)) } - logkit.Info("support plugins", zap.Strings("plugins", plugin.Plugins())) + logkit.Info("support plugins", zap.Strings("plugins", factory.Plugins())) logkit.Info("support handlers", zap.Strings("handlers", handler.Handlers())) logkit.Info("current use plugins", zap.Strings("plugins", c.Plugins)) for _, ct := range c.CategoryPlugins { @@ -104,7 +111,6 @@ func buildCapture(c *config.Config, ss []searcher.ISearcher, catSs map[number.Ca capture.WithSaveDir(c.SaveDir), capture.WithSeacher(searcher.NewCategorySearcher(ss, catSs)), capture.WithProcessor(processor.NewGroup(ps)), - capture.WithEnableLinkMode(c.SwitchConfig.EnableLinkMode), capture.WithExtraMediaExtList(c.ExtraMediaExts), ) return capture.New(opts...) @@ -129,7 +135,7 @@ func buildSearcher(plgs []string, m map[string]interface{}) ([]searcher.ISearche if !ok { args = struct{}{} } - plg, err := plugin.CreatePlugin(name, args) + plg, err := factory.CreatePlugin(name, args) if err != nil { return nil, fmt.Errorf("create plugin failed, name:%s, err:%w", name, err) } diff --git a/processor/handler/tag_padder_handler.go b/processor/handler/tag_padder_handler.go index 47f0939..4375a30 100644 --- a/processor/handler/tag_padder_handler.go +++ b/processor/handler/tag_padder_handler.go @@ -29,12 +29,26 @@ func (h *tagPadderHandler) generateNumberPrefixTag(fc *model.FileContext) (strin return sb.String(), true } +func (h *tagPadderHandler) rewriteOrAppendTag(fc *model.AvMeta, tagname string) { + isContained := false + for idx, item := range fc.Genres { + if strings.EqualFold(item, tagname) { + fc.Genres[idx] = tagname + isContained = true + } + } + if isContained { + return + } + fc.Genres = append(fc.Genres, tagname) +} + func (h *tagPadderHandler) Handle(ctx context.Context, fc *model.FileContext) error { //提取番号特有的tag fc.Meta.Genres = append(fc.Meta.Genres, fc.Number.GenerateTags()...) //提取番号前缀作为tag if tag, ok := h.generateNumberPrefixTag(fc); ok { - fc.Meta.Genres = append(fc.Meta.Genres, tag) + h.rewriteOrAppendTag(fc.Meta, tag) } fc.Meta.Genres = utils.DedupStringList(fc.Meta.Genres) return nil diff --git a/searcher/default_searcher.go b/searcher/default_searcher.go index b1a3929..6667a3f 100644 --- a/searcher/default_searcher.go +++ b/searcher/default_searcher.go @@ -7,10 +7,12 @@ import ( "strings" "time" "yamdc/client" + "yamdc/envflag" "yamdc/hasher" "yamdc/model" "yamdc/number" - "yamdc/searcher/plugin" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/meta" "yamdc/store" "yamdc/useragent" @@ -25,11 +27,11 @@ const ( type DefaultSearcher struct { name string ua string - invoker plugin.HTTPInvoker - plg plugin.IPlugin + invoker api.HTTPInvoker + plg api.IPlugin } -func MustNewDefaultSearcher(name string, plg plugin.IPlugin) ISearcher { +func MustNewDefaultSearcher(name string, plg api.IPlugin) ISearcher { s, err := NewDefaultSearcher(name, plg) if err != nil { panic(err) @@ -37,14 +39,14 @@ func MustNewDefaultSearcher(name string, plg plugin.IPlugin) ISearcher { return s } -func defaultInvoker() plugin.HTTPInvoker { +func defaultInvoker() api.HTTPInvoker { basicClient := client.NewClient() - return func(ctx *plugin.PluginContext, req *http.Request) (*http.Response, error) { + return func(ctx context.Context, req *http.Request) (*http.Response, error) { return basicClient.Do(req) } } -func NewDefaultSearcher(name string, plg plugin.IPlugin) (ISearcher, error) { +func NewDefaultSearcher(name string, plg api.IPlugin) (ISearcher, error) { invoker := plg.OnHTTPClientInit() if invoker == nil { invoker = defaultInvoker() @@ -72,7 +74,7 @@ func (p *DefaultSearcher) setDefaultHttpOptions(req *http.Request) error { return nil } -func (p *DefaultSearcher) decorateRequest(ctx *plugin.PluginContext, req *http.Request) error { +func (p *DefaultSearcher) decorateRequest(ctx context.Context, req *http.Request) error { if err := p.plg.OnDecorateRequest(ctx, req); err != nil { return err } @@ -82,7 +84,7 @@ func (p *DefaultSearcher) decorateRequest(ctx *plugin.PluginContext, req *http.R return nil } -func (p *DefaultSearcher) decorateImageRequest(ctx *plugin.PluginContext, req *http.Request) error { +func (p *DefaultSearcher) decorateImageRequest(ctx context.Context, req *http.Request) error { if err := p.plg.OnDecorateMediaRequest(ctx, req); err != nil { return err } @@ -92,21 +94,21 @@ func (p *DefaultSearcher) decorateImageRequest(ctx *plugin.PluginContext, req *h return nil } -func (p *DefaultSearcher) invokeHTTPRequest(ctx *plugin.PluginContext, req *http.Request) (*http.Response, error) { +func (p *DefaultSearcher) invokeHTTPRequest(ctx context.Context, req *http.Request) (*http.Response, error) { if err := p.decorateRequest(ctx, req); err != nil { return nil, fmt.Errorf("decorate request failed, err:%w", err) } return p.invoker(ctx, req) } -func (p *DefaultSearcher) onRetriveData(ctx context.Context, pctx *plugin.PluginContext, req *http.Request, number *number.Number) ([]byte, error) { +func (p *DefaultSearcher) onRetriveData(ctx context.Context, req *http.Request, number *number.Number) ([]byte, error) { key := p.name + ":" + number.GetNumberID() - return store.LoadData(ctx, key, defaultPageSearchCacheExpire, func() ([]byte, error) { - rsp, err := p.plg.OnHandleHTTPRequest(pctx, p.invokeHTTPRequest, req) + dataLoader := func() ([]byte, error) { + rsp, err := p.plg.OnHandleHTTPRequest(ctx, p.invokeHTTPRequest, req) if err != nil { return nil, fmt.Errorf("do request failed, err:%w", err) } - isSearchSucc, err := p.plg.OnPrecheckResponse(pctx, req, rsp) + isSearchSucc, err := p.plg.OnPrecheckResponse(ctx, req, rsp) if err != nil { return nil, fmt.Errorf("precheck responnse failed, err:%w", err) } @@ -122,28 +124,31 @@ func (p *DefaultSearcher) onRetriveData(ctx context.Context, pctx *plugin.Plugin return nil, fmt.Errorf("read body failed, err:%w", err) } return data, nil - }) + } + if !envflag.IsEnableSearchMetaCache() { + return dataLoader() + } + return store.LoadData(ctx, key, defaultPageSearchCacheExpire, dataLoader) } func (p *DefaultSearcher) Search(ctx context.Context, number *number.Number) (*model.AvMeta, bool, error) { - pctx := plugin.NewPluginContext(ctx) - pctx.SetNumberInfo(number) - ok, err := p.plg.OnPrecheckRequest(pctx, number) + ctx = meta.SetNumberId(ctx, number.GetNumberID()) + ok, err := p.plg.OnPrecheckRequest(ctx, number) if err != nil { return nil, false, fmt.Errorf("precheck failed, err:%w", err) } if !ok { return nil, false, nil } - req, err := p.plg.OnMakeHTTPRequest(pctx, number) + req, err := p.plg.OnMakeHTTPRequest(ctx, number) if err != nil { return nil, false, fmt.Errorf("make http request failed, err:%w", err) } - data, err := p.onRetriveData(ctx, pctx, req, number) + data, err := p.onRetriveData(ctx, req, number) if err != nil { return nil, false, err } - meta, decodeSucc, err := p.plg.OnDecodeHTTPData(pctx, data) + meta, decodeSucc, err := p.plg.OnDecodeHTTPData(ctx, data) if err != nil { return nil, false, fmt.Errorf("decode http data failed, err:%w", err) } @@ -153,7 +158,7 @@ func (p *DefaultSearcher) Search(ctx context.Context, number *number.Number) (*m //重建不规范的元数据 p.fixMeta(req, meta) //将远程数据保存到本地, 并替换文件key - p.storeImageData(pctx, meta) + p.storeImageData(ctx, meta) if err := p.verifyMeta(meta); err != nil { logutil.GetLogger(ctx).Error("verify meta not pass, treat as not found", zap.Error(err), zap.String("plugin", p.name)) return nil, false, nil @@ -204,7 +209,7 @@ func (p *DefaultSearcher) fixSingleURL(req *http.Request, input *string, prefix } } -func (p *DefaultSearcher) storeImageData(ctx *plugin.PluginContext, in *model.AvMeta) { +func (p *DefaultSearcher) storeImageData(ctx context.Context, in *model.AvMeta) { images := make([]string, 0, len(in.SampleImages)+2) if in.Cover != nil { images = append(images, in.Cover.Name) @@ -237,7 +242,7 @@ func (p *DefaultSearcher) storeImageData(ctx *plugin.PluginContext, in *model.Av in.SampleImages = rebuildSampleList } -func (p *DefaultSearcher) saveRemoteURLData(ctx *plugin.PluginContext, urls []string) map[string]string { +func (p *DefaultSearcher) saveRemoteURLData(ctx context.Context, urls []string) map[string]string { rs := make(map[string]string, len(urls)) for _, url := range urls { if len(url) == 0 { @@ -245,7 +250,7 @@ func (p *DefaultSearcher) saveRemoteURLData(ctx *plugin.PluginContext, urls []st } logger := logutil.GetLogger(context.Background()).With(zap.String("url", url)) key := hasher.ToSha1(url) - if ok, _ := store.IsDataExist(ctx.GetContext(), key); ok { + if ok, _ := store.IsDataExist(ctx, key); ok { rs[url] = key continue } @@ -254,7 +259,7 @@ func (p *DefaultSearcher) saveRemoteURLData(ctx *plugin.PluginContext, urls []st logger.Error("fetch image data failed", zap.Error(err)) continue } - err = store.PutData(ctx.GetContext(), key, data) + err = store.PutData(ctx, key, data) if err != nil { logger.Error("put image data to store failed", zap.Error(err)) } @@ -263,7 +268,7 @@ func (p *DefaultSearcher) saveRemoteURLData(ctx *plugin.PluginContext, urls []st return rs } -func (p *DefaultSearcher) fetchImageData(ctx *plugin.PluginContext, url string) ([]byte, error) { +func (p *DefaultSearcher) fetchImageData(ctx context.Context, url string) ([]byte, error) { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("make request for url:%s failed, err:%w", url, err) diff --git a/searcher/plugin/api/api.go b/searcher/plugin/api/api.go new file mode 100644 index 0000000..52cff57 --- /dev/null +++ b/searcher/plugin/api/api.go @@ -0,0 +1,21 @@ +package api + +import ( + "context" + "net/http" + "yamdc/model" + "yamdc/number" +) + +type HTTPInvoker func(ctx context.Context, req *http.Request) (*http.Response, error) + +type IPlugin interface { + OnHTTPClientInit() HTTPInvoker + OnPrecheckRequest(ctx context.Context, number *number.Number) (bool, error) + OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) + OnDecorateRequest(ctx context.Context, req *http.Request) error + OnHandleHTTPRequest(ctx context.Context, invoker HTTPInvoker, req *http.Request) (*http.Response, error) + OnPrecheckResponse(ctx context.Context, req *http.Request, rsp *http.Response) (bool, error) + OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) + OnDecorateMediaRequest(ctx context.Context, req *http.Request) error +} diff --git a/searcher/plugin/api/defaults.go b/searcher/plugin/api/defaults.go new file mode 100644 index 0000000..139b14b --- /dev/null +++ b/searcher/plugin/api/defaults.go @@ -0,0 +1,47 @@ +package api + +import ( + "context" + "fmt" + "net/http" + "yamdc/model" + "yamdc/number" +) + +type DefaultPlugin struct { +} + +func (p *DefaultPlugin) OnPrecheckRequest(ctx context.Context, number *number.Number) (bool, error) { + return true, nil +} + +func (p *DefaultPlugin) OnHTTPClientInit() HTTPInvoker { + return nil +} + +func (p *DefaultPlugin) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { + return nil, fmt.Errorf("no impl") +} + +func (p *DefaultPlugin) OnDecorateRequest(ctx context.Context, req *http.Request) error { + return nil +} + +func (p *DefaultPlugin) OnPrecheckResponse(ctx context.Context, req *http.Request, rsp *http.Response) (bool, error) { + if rsp.StatusCode == http.StatusNotFound { + return false, nil + } + return true, nil +} + +func (p *DefaultPlugin) OnHandleHTTPRequest(ctx context.Context, invoker HTTPInvoker, req *http.Request) (*http.Response, error) { + return invoker(ctx, req) +} + +func (p *DefaultPlugin) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { + return nil, false, fmt.Errorf("no impl") +} + +func (p *DefaultPlugin) OnDecorateMediaRequest(ctx context.Context, req *http.Request) error { + return nil +} diff --git a/searcher/plugin/api/domain_selector.go b/searcher/plugin/api/domain_selector.go new file mode 100644 index 0000000..aa4f3ac --- /dev/null +++ b/searcher/plugin/api/domain_selector.go @@ -0,0 +1,21 @@ +package api + +import "math/rand" + +func SelectDomain(in []string) (string, bool) { + if len(in) == 0 { + return "", false + } + if len(in) == 1 { + return in[0], true + } + return in[rand.Int()%len(in)], true +} + +func MustSelectDomain(in []string) string { + res, ok := SelectDomain(in) + if !ok { + panic("unable to select domain") + } + return res +} diff --git a/searcher/plugin/constant.go b/searcher/plugin/constant/constant.go similarity index 94% rename from searcher/plugin/constant.go rename to searcher/plugin/constant/constant.go index 8cc641a..1600bc7 100644 --- a/searcher/plugin/constant.go +++ b/searcher/plugin/constant/constant.go @@ -1,4 +1,4 @@ -package plugin +package constant const ( SSJavBus = "javbus" diff --git a/searcher/plugin/factory/factory.go b/searcher/plugin/factory/factory.go new file mode 100644 index 0000000..1f92bab --- /dev/null +++ b/searcher/plugin/factory/factory.go @@ -0,0 +1,37 @@ +package factory + +import ( + "fmt" + "sort" + "yamdc/searcher/plugin/api" +) + +type CreatorFunc func(args interface{}) (api.IPlugin, error) + +var mp = make(map[string]CreatorFunc) + +func Register(name string, fn CreatorFunc) { + mp[name] = fn +} + +func CreatePlugin(name string, args interface{}) (api.IPlugin, error) { + cr, ok := mp[name] + if !ok { + return nil, fmt.Errorf("plugin:%s not found", name) + } + return cr(args) +} + +func PluginToCreator(plg api.IPlugin) CreatorFunc { + return func(args interface{}) (api.IPlugin, error) { + return plg, nil + } +} + +func Plugins() []string { + rs := make([]string, 0, len(mp)) + for k := range mp { + rs = append(rs, k) + } + return sort.StringSlice(rs) +} diff --git a/searcher/plugin/18av.go b/searcher/plugin/impl/18av.go similarity index 73% rename from searcher/plugin/18av.go rename to searcher/plugin/impl/18av.go index 33e9a40..af1af5d 100644 --- a/searcher/plugin/18av.go +++ b/searcher/plugin/impl/18av.go @@ -1,6 +1,7 @@ -package plugin +package impl import ( + "context" "fmt" "net/http" "strings" @@ -8,20 +9,25 @@ import ( "yamdc/number" "yamdc/searcher/decoder" "yamdc/searcher/parser" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" + "yamdc/searcher/plugin/meta" + "yamdc/searcher/plugin/twostep" ) type av18 struct { - DefaultPlugin + api.DefaultPlugin } -func (p *av18) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { +func (p *av18) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { uri := fmt.Sprintf("https://18av.me/cn/search.php?kw_type=key&kw=%s", number.GetNumberID()) return http.NewRequest(http.MethodGet, uri, nil) } -func (p *av18) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req *http.Request) (*http.Response, error) { - xctx := &XPathTwoStepContext{ - Ps: []*XPathPair{ +func (p *av18) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, req *http.Request) (*http.Response, error) { + xctx := &twostep.XPathTwoStepContext{ + Ps: []*twostep.XPathPair{ { Name: "read-link", XPath: `//div[@class="content flex-columns small px-2"]/span[@class="title"]/a/@href`, @@ -31,8 +37,8 @@ func (p *av18) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req XPath: `//div[@class="content flex-columns small px-2"]/span[@class="title"]/a/text()`, }, }, - LinkSelector: func(ps []*XPathPair) (string, bool, error) { - number := strings.ToUpper(ctx.MustGetNumberInfo().GetNumberID()) + LinkSelector: func(ps []*twostep.XPathPair) (string, bool, error) { + number := strings.ToUpper(meta.GetNumberId(ctx)) linkList := ps[0].Result titleList := ps[1].Result for idx, link := range linkList { @@ -47,7 +53,7 @@ func (p *av18) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req CheckResultCountMatch: true, LinkPrefix: "https://18av.me/cn", } - return HandleXPathTwoStepSearch(ctx, invoker, req, xctx) + return twostep.HandleXPathTwoStepSearch(ctx, invoker, req, xctx) } func (p *av18) coverParser(in string) string { @@ -58,7 +64,7 @@ func (p *av18) plotParser(in string) string { return strings.TrimSpace(strings.TrimLeft(in, "简介:")) } -func (p *av18) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *av18) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := decoder.XPathHtmlDecoder{ NumberExpr: `//div[@class="px-0 flex-columns"]/div[@class="number"]/text()`, TitleExpr: `//div[@class="d-flex px-3 py-2 name col bg-w"]/h1[@class="h4 b"]/text()`, @@ -78,8 +84,8 @@ func (p *av18) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, meta, err := dec.DecodeHTML(data, decoder.WithCoverParser(p.coverParser), decoder.WithPlotParser(p.plotParser), - decoder.WithDurationParser(parser.DefaultDurationParser(ctx.GetContext())), - decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx.GetContext())), + decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), + decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx)), ) if err != nil { return nil, false, err @@ -91,5 +97,5 @@ func (p *av18) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, } func init() { - Register(SS18AV, PluginToCreator(&av18{})) + factory.Register(constant.SS18AV, factory.PluginToCreator(&av18{})) } diff --git a/searcher/plugin/airav/airav.go b/searcher/plugin/impl/airav/airav.go similarity index 76% rename from searcher/plugin/airav/airav.go rename to searcher/plugin/impl/airav/airav.go index 24d65d6..9d63382 100644 --- a/searcher/plugin/airav/airav.go +++ b/searcher/plugin/impl/airav/airav.go @@ -1,6 +1,7 @@ package airav import ( + "context" "encoding/json" "fmt" "net/http" @@ -8,17 +9,19 @@ import ( "yamdc/model" "yamdc/number" "yamdc/searcher/parser" - "yamdc/searcher/plugin" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" "github.com/xxxsen/common/logutil" "go.uber.org/zap" ) type airav struct { - plugin.DefaultPlugin + api.DefaultPlugin } -func (p *airav) OnMakeHTTPRequest(ctx *plugin.PluginContext, number *number.Number) (*http.Request, error) { +func (p *airav) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://www.airav.wiki/api/video/barcode/%s?lng=zh-TW", number.GetNumberID()), nil) if err != nil { return nil, err @@ -26,7 +29,7 @@ func (p *airav) OnMakeHTTPRequest(ctx *plugin.PluginContext, number *number.Numb return req, nil } -func (p *airav) OnDecodeHTTPData(ctx *plugin.PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *airav) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { vdata := &VideoData{} if err := json.Unmarshal(data, vdata); err != nil { return nil, false, fmt.Errorf("decode json data failed, err:%w", err) @@ -38,7 +41,7 @@ func (p *airav) OnDecodeHTTPData(ctx *plugin.PluginContext, data []byte) (*model return nil, false, nil } if vdata.Count > 1 { - logutil.GetLogger(ctx.GetContext()).Warn("more than one result, may cause data mismatch", zap.Int("count", vdata.Count)) + logutil.GetLogger(ctx).Warn("more than one result, may cause data mismatch", zap.Int("count", vdata.Count)) } result := vdata.Result avdata := &model.AvMeta{ @@ -46,7 +49,7 @@ func (p *airav) OnDecodeHTTPData(ctx *plugin.PluginContext, data []byte) (*model Title: result.Name, Plot: result.Description, Actors: p.readActors(&result), - ReleaseDate: parser.DefaultReleaseDateParser(ctx.GetContext())(result.PublishDate), + ReleaseDate: parser.DefaultReleaseDateParser(ctx)(result.PublishDate), Studio: p.readStudio(&result), Genres: p.readGenres(&result), Cover: &model.File{ @@ -91,5 +94,5 @@ func (p *airav) readActors(result *Result) []string { } func init() { - plugin.Register(plugin.SSAirav, plugin.PluginToCreator(&airav{})) + factory.Register(constant.SSAirav, factory.PluginToCreator(&airav{})) } diff --git a/searcher/plugin/airav/model.go b/searcher/plugin/impl/airav/model.go similarity index 100% rename from searcher/plugin/airav/model.go rename to searcher/plugin/impl/airav/model.go diff --git a/searcher/plugin/avsox.go b/searcher/plugin/impl/avsox.go similarity index 80% rename from searcher/plugin/avsox.go rename to searcher/plugin/impl/avsox.go index 7927ef6..5dc279a 100644 --- a/searcher/plugin/avsox.go +++ b/searcher/plugin/impl/avsox.go @@ -1,6 +1,7 @@ -package plugin +package impl import ( + "context" "fmt" "net/http" "strings" @@ -8,6 +9,10 @@ import ( "yamdc/number" "yamdc/searcher/decoder" "yamdc/searcher/parser" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" + "yamdc/searcher/plugin/meta" "yamdc/searcher/utils" "github.com/xxxsen/common/logutil" @@ -19,18 +24,17 @@ const ( ) type avsox struct { - DefaultPlugin + api.DefaultPlugin } -func (p *avsox) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { +func (p *avsox) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { return http.NewRequest(http.MethodGet, "https://avsox.click", nil) //返回一个假的request } -func (p *avsox) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, _ *http.Request) (*http.Response, error) { - number := ctx.MustGetNumberInfo() - num := strings.ToUpper(number.GetNumberID()) +func (p *avsox) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, _ *http.Request) (*http.Response, error) { + num := strings.ToUpper(meta.GetNumberId(ctx)) tryList := p.generateTryList(num) - logger := logutil.GetLogger(ctx.GetContext()).With(zap.String("plugin", "avsox")) + logger := logutil.GetLogger(ctx).With(zap.String("plugin", "avsox")) logger.Debug("build try list succ", zap.Int("count", len(tryList)), zap.Strings("list", tryList)) var link string var ok bool @@ -70,7 +74,7 @@ func (p *avsox) generateTryList(num string) []string { return tryList } -func (p *avsox) trySearchByNumber(ctx *PluginContext, invoker HTTPInvoker, number string) (string, bool, error) { +func (p *avsox) trySearchByNumber(ctx context.Context, invoker api.HTTPInvoker, number string) (string, bool, error) { req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("https://avsox.click/cn/search/%s", number), nil) if err != nil { return "", false, err @@ -100,7 +104,7 @@ func (p *avsox) trySearchByNumber(ctx *PluginContext, invoker HTTPInvoker, numbe return res[0], true, nil } -func (p *avsox) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *avsox) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := decoder.XPathHtmlDecoder{ NumberExpr: `//span[contains(text(),"识别码:")]/../span[2]/text()`, TitleExpr: `/html/body/div[2]/h3/text()`, @@ -118,8 +122,8 @@ func (p *avsox) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta SampleImageListExpr: "", } meta, err := dec.DecodeHTML(data, - decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx.GetContext())), - decoder.WithDurationParser(parser.DefaultDurationParser(ctx.GetContext())), + decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx)), + decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), decoder.WithDefaultStringProcessor(strings.TrimSpace), ) if err != nil { @@ -133,5 +137,5 @@ func (p *avsox) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta } func init() { - Register(SSAvsox, PluginToCreator(&avsox{})) + factory.Register(constant.SSAvsox, factory.PluginToCreator(&avsox{})) } diff --git a/searcher/plugin/caribpr.go b/searcher/plugin/impl/caribpr.go similarity index 75% rename from searcher/plugin/caribpr.go rename to searcher/plugin/impl/caribpr.go index 585e194..9ce421d 100644 --- a/searcher/plugin/caribpr.go +++ b/searcher/plugin/impl/caribpr.go @@ -1,4 +1,4 @@ -package plugin +package impl import ( "context" @@ -10,6 +10,10 @@ import ( "yamdc/model" "yamdc/number" "yamdc/searcher/decoder" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" + "yamdc/searcher/plugin/meta" putils "yamdc/searcher/utils" "yamdc/utils" @@ -20,10 +24,10 @@ import ( ) type caribpr struct { - DefaultPlugin + api.DefaultPlugin } -func (p *caribpr) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { +func (p *caribpr) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { uri := fmt.Sprintf("https://www.caribbeancompr.com/moviepages/%s/index.html", number.GetNumberID()) req, err := http.NewRequest(http.MethodGet, uri, nil) return req, err @@ -51,7 +55,7 @@ func (p *caribpr) decodeReleaseDate(ctx context.Context) decoder.NumberParseFunc } } -func (p *caribpr) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *caribpr) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { reader := transform.NewReader(strings.NewReader(string(data)), japanese.EUCJP.NewDecoder()) data, err := io.ReadAll(reader) if err != nil { @@ -71,19 +75,19 @@ func (p *caribpr) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMe PosterExpr: ``, SampleImageListExpr: `//div[@class='movie-gallery']/div[@class='section is-wide']/div[2]/div[@class='grid-item']/div/a/@href`, } - meta, err := dec.DecodeHTML(data, - decoder.WithDurationParser(p.decodeDuration(ctx.GetContext())), - decoder.WithReleaseDateParser(p.decodeReleaseDate(ctx.GetContext())), + metadata, err := dec.DecodeHTML(data, + decoder.WithDurationParser(p.decodeDuration(ctx)), + decoder.WithReleaseDateParser(p.decodeReleaseDate(ctx)), ) if err != nil { return nil, false, err } - meta.Number = ctx.MustGetNumberInfo().GetNumberID() - meta.Cover.Name = fmt.Sprintf("https://www.caribbeancompr.com/moviepages/%s/images/l_l.jpg", meta.Number) - putils.EnableDataTranslate(meta) - return meta, true, nil + metadata.Number = meta.GetNumberId(ctx) + metadata.Cover.Name = fmt.Sprintf("https://www.caribbeancompr.com/moviepages/%s/images/l_l.jpg", metadata.Number) + putils.EnableDataTranslate(metadata) + return metadata, true, nil } func init() { - Register(SSCaribpr, PluginToCreator(&caribpr{})) + factory.Register(constant.SSCaribpr, factory.PluginToCreator(&caribpr{})) } diff --git a/searcher/plugin/fc2.go b/searcher/plugin/impl/fc2.go similarity index 79% rename from searcher/plugin/fc2.go rename to searcher/plugin/impl/fc2.go index 862eab1..d29f0df 100644 --- a/searcher/plugin/fc2.go +++ b/searcher/plugin/impl/fc2.go @@ -1,4 +1,4 @@ -package plugin +package impl import ( "context" @@ -11,6 +11,10 @@ import ( "yamdc/model" "yamdc/number" "yamdc/searcher/decoder" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" + "yamdc/searcher/plugin/meta" putils "yamdc/searcher/utils" "github.com/xxxsen/common/logutil" @@ -20,10 +24,10 @@ import ( var defaultFc2NumberParser = regexp.MustCompile(`^fc2.*?(\d+)$`) type fc2 struct { - DefaultPlugin + api.DefaultPlugin } -func (p *fc2) OnMakeHTTPRequest(ctx *PluginContext, n *number.Number) (*http.Request, error) { +func (p *fc2) OnMakeHTTPRequest(ctx context.Context, n *number.Number) (*http.Request, error) { number := strings.ToLower(n.GetNumberID()) res := defaultFc2NumberParser.FindStringSubmatch(number) if len(res) != 2 { @@ -74,7 +78,7 @@ func (p *fc2) decodeReleaseDate(ctx context.Context) decoder.NumberParseFunc { } } -func (p *fc2) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *fc2) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := decoder.XPathHtmlDecoder{ NumberExpr: ``, TitleExpr: `/html/head/title/text()`, @@ -89,18 +93,18 @@ func (p *fc2) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, PosterExpr: `//div[@class='items_article_MainitemThumb']/span/img/@src`, //这东西就一张封面图, 直接当海报得了 SampleImageListExpr: `//ul[@class="items_article_SampleImagesArea"]/li/a/@href`, } - meta, err := dec.DecodeHTML(data, - decoder.WithDurationParser(p.decodeDuration(ctx.GetContext())), - decoder.WithReleaseDateParser(p.decodeReleaseDate(ctx.GetContext())), + metadata, err := dec.DecodeHTML(data, + decoder.WithDurationParser(p.decodeDuration(ctx)), + decoder.WithReleaseDateParser(p.decodeReleaseDate(ctx)), ) if err != nil { return nil, false, err } - meta.Number = ctx.MustGetNumberInfo().GetNumberID() - putils.EnableDataTranslate(meta) - return meta, true, nil + metadata.Number = meta.GetNumberId(ctx) + putils.EnableDataTranslate(metadata) + return metadata, true, nil } func init() { - Register(SSFc2, PluginToCreator(&fc2{})) + factory.Register(constant.SSFc2, factory.PluginToCreator(&fc2{})) } diff --git a/searcher/plugin/freejavbt.go b/searcher/plugin/impl/freejavbt.go similarity index 70% rename from searcher/plugin/freejavbt.go rename to searcher/plugin/impl/freejavbt.go index be71792..b20b2d8 100644 --- a/searcher/plugin/freejavbt.go +++ b/searcher/plugin/impl/freejavbt.go @@ -1,24 +1,29 @@ -package plugin +package impl import ( + "context" "net/http" "yamdc/model" "yamdc/number" "yamdc/searcher/decoder" "yamdc/searcher/parser" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" + "yamdc/searcher/plugin/meta" putils "yamdc/searcher/utils" ) type freejavbt struct { - DefaultPlugin + api.DefaultPlugin } -func (p *freejavbt) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { +func (p *freejavbt) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { uri := "https://freejavbt.com/zh/" + number.GetNumberID() return http.NewRequest(http.MethodGet, uri, nil) } -func (p *freejavbt) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *freejavbt) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := decoder.XPathHtmlDecoder{ NumberExpr: "", TitleExpr: `//h1[@class="text-white"]/strong/text()`, @@ -36,17 +41,17 @@ func (p *freejavbt) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.Av SampleImageListExpr: `//div[@class="preview"]/a/img/@data-src`, } res, err := dec.DecodeHTML(data, - decoder.WithDurationParser(parser.DefaultDurationParser(ctx.GetContext())), - decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx.GetContext())), + decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), + decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx)), ) if err != nil { return nil, false, err } - res.Number = ctx.MustGetNumberInfo().GetNumberID() + res.Number = meta.GetNumberId(ctx) putils.EnableDataTranslate(res) return res, true, nil } func init() { - Register(SSFreeJavBt, PluginToCreator(&freejavbt{})) + factory.Register(constant.SSFreeJavBt, factory.PluginToCreator(&freejavbt{})) } diff --git a/searcher/plugin/jav321.go b/searcher/plugin/impl/jav321.go similarity index 81% rename from searcher/plugin/jav321.go rename to searcher/plugin/impl/jav321.go index 7dd9b07..e2b5db2 100644 --- a/searcher/plugin/jav321.go +++ b/searcher/plugin/impl/jav321.go @@ -1,6 +1,7 @@ -package plugin +package impl import ( + "context" "net/http" "net/url" "strconv" @@ -9,14 +10,17 @@ import ( "yamdc/number" "yamdc/searcher/decoder" "yamdc/searcher/parser" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" putils "yamdc/searcher/utils" ) type jav321 struct { - DefaultPlugin + api.DefaultPlugin } -func (p *jav321) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { +func (p *jav321) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { data := url.Values{} data.Set("sn", number.GetNumberID()) body := data.Encode() @@ -34,7 +38,7 @@ func (s *jav321) defaultStringProcessor(v string) string { return strings.TrimSpace(v) } -func (p *jav321) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *jav321) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := &decoder.XPathHtmlDecoder{ NumberExpr: `//b[contains(text(),"品番")]/following-sibling::node()`, TitleExpr: `/html/body/div[2]/div[1]/div[1]/div[1]/h3/text()`, @@ -52,8 +56,8 @@ func (p *jav321) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMet } rs, err := dec.DecodeHTML(data, decoder.WithDefaultStringProcessor(p.defaultStringProcessor), - decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx.GetContext())), - decoder.WithDurationParser(parser.DefaultDurationParser(ctx.GetContext())), + decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx)), + decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), ) if err != nil { return nil, false, err @@ -63,5 +67,5 @@ func (p *jav321) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMet } func init() { - Register(SSJav321, PluginToCreator(&jav321{})) + factory.Register(constant.SSJav321, factory.PluginToCreator(&jav321{})) } diff --git a/searcher/plugin/javbus.go b/searcher/plugin/impl/javbus.go similarity index 74% rename from searcher/plugin/javbus.go rename to searcher/plugin/impl/javbus.go index d80b76b..2b49513 100644 --- a/searcher/plugin/javbus.go +++ b/searcher/plugin/impl/javbus.go @@ -1,24 +1,33 @@ -package plugin +package impl import ( + "context" + "fmt" "net/http" "yamdc/model" "yamdc/number" "yamdc/searcher/decoder" "yamdc/searcher/parser" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" putils "yamdc/searcher/utils" ) +var defaultJavBusDomainList = []string{ + "www.javbus.com", +} + type javbus struct { - DefaultPlugin + api.DefaultPlugin } -func (p *javbus) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { - url := "https://www.javbus.com/" + number.GetNumberID() +func (p *javbus) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { + url := fmt.Sprintf("https://%s/%s", api.MustSelectDomain(defaultJavBusDomainList), number.GetNumberID()) return http.NewRequest(http.MethodGet, url, nil) } -func (p *javbus) OnDecorateRequest(ctx *PluginContext, req *http.Request) error { +func (p *javbus) OnDecorateRequest(ctx context.Context, req *http.Request) error { req.AddCookie(&http.Cookie{ Name: "existmag", Value: "mag", @@ -37,7 +46,7 @@ func (p *javbus) OnDecorateRequest(ctx *PluginContext, req *http.Request) error return nil } -func (p *javbus) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *javbus) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := decoder.XPathHtmlDecoder{ NumberExpr: `//div[@class="row movie"]/div[@class="col-md-3 info"]/p[span[contains(text(),'識別碼:')]]/span[2]/text()`, TitleExpr: `//div[@class="container"]/h3`, @@ -54,8 +63,8 @@ func (p *javbus) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMet SampleImageListExpr: `//div[@id="sample-waterfall"]/a[@class="sample-box"]/@href`, } rs, err := dec.DecodeHTML(data, - decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx.GetContext())), - decoder.WithDurationParser(parser.DefaultDurationParser(ctx.GetContext())), + decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx)), + decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), ) if err != nil { return nil, false, err @@ -65,5 +74,5 @@ func (p *javbus) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMet } func init() { - Register(SSJavBus, PluginToCreator(&javbus{})) + factory.Register(constant.SSJavBus, factory.PluginToCreator(&javbus{})) } diff --git a/searcher/plugin/javdb.go b/searcher/plugin/impl/javdb.go similarity index 71% rename from searcher/plugin/javdb.go rename to searcher/plugin/impl/javdb.go index 44a2f43..e5a04cb 100644 --- a/searcher/plugin/javdb.go +++ b/searcher/plugin/impl/javdb.go @@ -1,27 +1,33 @@ -package plugin +package impl import ( + "context" "fmt" "net/http" "yamdc/model" "yamdc/number" "yamdc/searcher/decoder" "yamdc/searcher/parser" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" + "yamdc/searcher/plugin/meta" + "yamdc/searcher/plugin/twostep" "yamdc/searcher/utils" ) type javdb struct { - DefaultPlugin + api.DefaultPlugin } -func (p *javdb) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { +func (p *javdb) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { link := fmt.Sprintf("https://javdb.com/search?q=%s&f=all", number.GetNumberID()) return http.NewRequest(http.MethodGet, link, nil) } -func (p *javdb) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req *http.Request) (*http.Response, error) { - return HandleXPathTwoStepSearch(ctx, invoker, req, &XPathTwoStepContext{ - Ps: []*XPathPair{ +func (p *javdb) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, req *http.Request) (*http.Response, error) { + return twostep.HandleXPathTwoStepSearch(ctx, invoker, req, &twostep.XPathTwoStepContext{ + Ps: []*twostep.XPathPair{ { Name: "read-link", XPath: `//div[@class="movie-list h cols-4 vcols-8"]/div[@class="item"]/a/@href`, @@ -31,10 +37,10 @@ func (p *javdb) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req XPath: `//div[@class="movie-list h cols-4 vcols-8"]/div[@class="item"]/a/div[@class="video-title"]/strong`, }, }, - LinkSelector: func(ps []*XPathPair) (string, bool, error) { + LinkSelector: func(ps []*twostep.XPathPair) (string, bool, error) { linklist := ps[0].Result numberlist := ps[1].Result - num := number.GetCleanID(ctx.MustGetNumberInfo().GetNumberID()) + num := number.GetCleanID(meta.GetNumberId(ctx)) for idx, numberItem := range numberlist { link := linklist[idx] if number.GetCleanID(numberItem) == num { @@ -49,7 +55,7 @@ func (p *javdb) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req }) } -func (p *javdb) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *javdb) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := decoder.XPathHtmlDecoder{ NumberExpr: `//a[@class="button is-white copy-to-clipboard"]/@data-clipboard-text`, TitleExpr: `//h2[@class="title is-4"]/strong[@class="current-title"]`, @@ -67,8 +73,8 @@ func (p *javdb) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta SampleImageListExpr: `//div[@class="tile-images preview-images"]/a[@class="tile-item"]/@href`, } meta, err := dec.DecodeHTML(data, - decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx.GetContext())), - decoder.WithDurationParser(parser.DefaultDurationParser(ctx.GetContext())), + decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx)), + decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), ) if err != nil { return nil, false, err @@ -81,5 +87,5 @@ func (p *javdb) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta } func init() { - Register(SSJavDB, PluginToCreator(&javdb{})) + factory.Register(constant.SSJavDB, factory.PluginToCreator(&javdb{})) } diff --git a/searcher/plugin/javhoo.go b/searcher/plugin/impl/javhoo.go similarity index 78% rename from searcher/plugin/javhoo.go rename to searcher/plugin/impl/javhoo.go index febb538..fdae462 100644 --- a/searcher/plugin/javhoo.go +++ b/searcher/plugin/impl/javhoo.go @@ -1,25 +1,29 @@ -package plugin +package impl import ( + "context" "fmt" "net/http" "yamdc/model" "yamdc/number" "yamdc/searcher/decoder" "yamdc/searcher/parser" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" putils "yamdc/searcher/utils" ) type javhoo struct { - DefaultPlugin + api.DefaultPlugin } -func (p *javhoo) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { +func (p *javhoo) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { uri := fmt.Sprintf("https://www.javhoo.com/av/%s", number.GetNumberID()) return http.NewRequest(http.MethodGet, uri, nil) } -func (p *javhoo) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *javhoo) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := decoder.XPathHtmlDecoder{ NumberExpr: `//div[@class="project_info"]/p/span[@class="categories"]/text()`, TitleExpr: `//header[@class="article-header"]/h1[@class="article-title"]/text()`, @@ -37,8 +41,8 @@ func (p *javhoo) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMet SampleImageListExpr: `//div[@id="sample-box"]/div/a/@href`, } meta, err := dec.DecodeHTML(data, - decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx.GetContext())), - decoder.WithDurationParser(parser.DefaultDurationParser(ctx.GetContext())), + decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx)), + decoder.WithDurationParser(parser.DefaultDurationParser(ctx)), ) if err != nil { return nil, false, err @@ -51,5 +55,5 @@ func (p *javhoo) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMet } func init() { - Register(SSJavhoo, PluginToCreator(&javhoo{})) + factory.Register(constant.SSJavhoo, factory.PluginToCreator(&javhoo{})) } diff --git a/searcher/plugin/njav.go b/searcher/plugin/impl/njav.go similarity index 74% rename from searcher/plugin/njav.go rename to searcher/plugin/impl/njav.go index 6fce230..34cd155 100644 --- a/searcher/plugin/njav.go +++ b/searcher/plugin/impl/njav.go @@ -1,6 +1,7 @@ -package plugin +package impl import ( + "context" "fmt" "net/http" "strings" @@ -8,23 +9,28 @@ import ( "yamdc/number" "yamdc/searcher/decoder" "yamdc/searcher/parser" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" + "yamdc/searcher/plugin/meta" + "yamdc/searcher/plugin/twostep" ) type njav struct { - DefaultPlugin + api.DefaultPlugin } -func (p *njav) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { +func (p *njav) OnMakeHTTPRequest(ctx context.Context, number *number.Number) (*http.Request, error) { nid := number.GetNumberID() nid = strings.ReplaceAll(nid, "_", "-") //将下划线替换为中划线 uri := fmt.Sprintf("https://njavtv.com/cn/search/%s", nid) return http.NewRequest(http.MethodGet, uri, nil) } -func (p *njav) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req *http.Request) (*http.Response, error) { - cleanNumberId := strings.ToUpper(number.GetCleanID(ctx.MustGetNumberInfo().GetNumberID())) - return HandleXPathTwoStepSearch(ctx, invoker, req, &XPathTwoStepContext{ - Ps: []*XPathPair{ +func (p *njav) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, req *http.Request) (*http.Response, error) { + cleanNumberId := strings.ToUpper(number.GetCleanID(meta.GetNumberId(ctx))) + return twostep.HandleXPathTwoStepSearch(ctx, invoker, req, &twostep.XPathTwoStepContext{ + Ps: []*twostep.XPathPair{ { Name: "links", XPath: `//div[@class="my-2 text-sm text-nord4 truncate"]/a[@class="text-secondary group-hover:text-primary"]/@href`, @@ -34,7 +40,7 @@ func (p *njav) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req XPath: `//div[@class="my-2 text-sm text-nord4 truncate"]/a[@class="text-secondary group-hover:text-primary"]/text()`, }, }, - LinkSelector: func(ps []*XPathPair) (string, bool, error) { + LinkSelector: func(ps []*twostep.XPathPair) (string, bool, error) { links := ps[0].Result titles := ps[1].Result for i, link := range links { @@ -52,7 +58,7 @@ func (p *njav) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req } -func (p *njav) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *njav) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := decoder.XPathHtmlDecoder{ NumberExpr: `//div[@class="text-secondary" and contains(span[text()], "番号:")]/span[@class="font-medium"]/text()`, TitleExpr: `//div[@class="text-secondary" and contains(span[text()], "标题:")]/span[@class="font-medium"]/text()`, @@ -69,7 +75,7 @@ func (p *njav) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, PosterExpr: "", SampleImageListExpr: "", } - meta, err := dec.DecodeHTML(data, decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx.GetContext()))) + meta, err := dec.DecodeHTML(data, decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx))) if err != nil { return nil, false, err } @@ -80,5 +86,5 @@ func (p *njav) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, } func init() { - Register(SSNJav, PluginToCreator(&njav{})) + factory.Register(constant.SSNJav, factory.PluginToCreator(&njav{})) } diff --git a/searcher/plugin/tktube.go b/searcher/plugin/impl/tktube.go similarity index 65% rename from searcher/plugin/tktube.go rename to searcher/plugin/impl/tktube.go index fc88189..5f01891 100644 --- a/searcher/plugin/tktube.go +++ b/searcher/plugin/impl/tktube.go @@ -1,6 +1,7 @@ -package plugin +package impl import ( + "context" "fmt" "net/http" "strings" @@ -8,22 +9,27 @@ import ( "yamdc/number" "yamdc/searcher/decoder" "yamdc/searcher/parser" + "yamdc/searcher/plugin/api" + "yamdc/searcher/plugin/constant" + "yamdc/searcher/plugin/factory" + "yamdc/searcher/plugin/meta" + "yamdc/searcher/plugin/twostep" ) type tktube struct { - DefaultPlugin + api.DefaultPlugin } -func (p *tktube) OnMakeHTTPRequest(ctx *PluginContext, n *number.Number) (*http.Request, error) { +func (p *tktube) OnMakeHTTPRequest(ctx context.Context, n *number.Number) (*http.Request, error) { nid := strings.ReplaceAll(n.GetNumberID(), "-", "--") uri := fmt.Sprintf("https://tktube.com/zh/search/%s/", nid) return http.NewRequest(http.MethodGet, uri, nil) } -func (p *tktube) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req *http.Request) (*http.Response, error) { - numberId := strings.ToUpper(ctx.MustGetNumberInfo().GetNumberID()) - return HandleXPathTwoStepSearch(ctx, invoker, req, &XPathTwoStepContext{ - Ps: []*XPathPair{ +func (p *tktube) OnHandleHTTPRequest(ctx context.Context, invoker api.HTTPInvoker, req *http.Request) (*http.Response, error) { + numberId := strings.ToUpper(meta.GetNumberId(ctx)) + return twostep.HandleXPathTwoStepSearch(ctx, invoker, req, &twostep.XPathTwoStepContext{ + Ps: []*twostep.XPathPair{ { Name: "links", XPath: `//div[@id="list_videos_videos_list_search_result_items"]/div/a/@href`, @@ -33,7 +39,7 @@ func (p *tktube) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, re XPath: `//div[@id="list_videos_videos_list_search_result_items"]/div/a/strong[@class="title"]/text()`, }, }, - LinkSelector: func(ps []*XPathPair) (string, bool, error) { + LinkSelector: func(ps []*twostep.XPathPair) (string, bool, error) { links := ps[0].Result names := ps[1].Result for i := 0; i < len(links); i++ { @@ -49,7 +55,7 @@ func (p *tktube) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, re }) } -func (p *tktube) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { +func (p *tktube) OnDecodeHTTPData(ctx context.Context, data []byte) (*model.AvMeta, bool, error) { dec := decoder.XPathHtmlDecoder{ TitleExpr: `//div[@class="headline"]/h1/text()`, PlotExpr: "", @@ -65,17 +71,17 @@ func (p *tktube) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMet PosterExpr: "", SampleImageListExpr: "", } - meta, err := dec.DecodeHTML(data, - decoder.WithDurationParser(parser.DefaultHHMMSSDurationParser(ctx.GetContext())), - decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx.GetContext())), + res, err := dec.DecodeHTML(data, + decoder.WithDurationParser(parser.DefaultHHMMSSDurationParser(ctx)), + decoder.WithReleaseDateParser(parser.DefaultReleaseDateParser(ctx)), ) if err != nil { return nil, false, err } - meta.Number = ctx.MustGetNumberInfo().GetNumberID() - return meta, true, nil + res.Number = meta.GetNumberId(ctx) + return res, true, nil } func init() { - Register(SSTKTube, PluginToCreator(&tktube{})) + factory.Register(constant.SSTKTube, factory.PluginToCreator(&tktube{})) } diff --git a/searcher/plugin/meta/meta.go b/searcher/plugin/meta/meta.go new file mode 100644 index 0000000..a9d1324 --- /dev/null +++ b/searcher/plugin/meta/meta.go @@ -0,0 +1,22 @@ +package meta + +import "context" + +type numberIdKeyType struct{} + +var ( + defaultNumberIdKey = numberIdKeyType{} +) + +func SetNumberId(ctx context.Context, nid string) context.Context { + ctx = context.WithValue(ctx, defaultNumberIdKey, nid) + return ctx +} + +func GetNumberId(ctx context.Context) string { + nid := ctx.Value(defaultNumberIdKey) + if nid == nil { + return "" + } + return nid.(string) +} diff --git a/searcher/plugin/plugin.go b/searcher/plugin/plugin.go deleted file mode 100644 index f2b7ee4..0000000 --- a/searcher/plugin/plugin.go +++ /dev/null @@ -1,133 +0,0 @@ -package plugin - -import ( - "context" - "fmt" - "net/http" - "sort" - "yamdc/model" - "yamdc/number" -) - -const ( - defaultNumberInfoKey = "plugin_ctx_number_info" -) - -type PluginContext struct { - ctx context.Context - attach map[string]interface{} -} - -func NewPluginContext(ctx context.Context) *PluginContext { - return &PluginContext{ - ctx: ctx, - attach: make(map[string]interface{}), - } -} - -func (s *PluginContext) GetContext() context.Context { - return s.ctx -} - -func (s *PluginContext) SetNumberInfo(v *number.Number) { - s.setKey(defaultNumberInfoKey, v) -} - -func (s *PluginContext) MustGetNumberInfo() *number.Number { - if v, ok := s.getKey(defaultNumberInfoKey); ok { - return v.(*number.Number) - } - panic("number info not found") -} - -func (s *PluginContext) setKey(key string, val interface{}) { - s.attach[key] = val -} - -func (s *PluginContext) getKey(key string) (interface{}, bool) { - v, ok := s.attach[key] - return v, ok -} - -type HTTPInvoker func(ctx *PluginContext, req *http.Request) (*http.Response, error) - -type IPlugin interface { - OnHTTPClientInit() HTTPInvoker - OnPrecheckRequest(ctx *PluginContext, number *number.Number) (bool, error) - OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) - OnDecorateRequest(ctx *PluginContext, req *http.Request) error - OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req *http.Request) (*http.Response, error) - OnPrecheckResponse(ctx *PluginContext, req *http.Request, rsp *http.Response) (bool, error) - OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) - OnDecorateMediaRequest(ctx *PluginContext, req *http.Request) error -} - -var _ IPlugin = &DefaultPlugin{} - -type DefaultPlugin struct { -} - -func (p *DefaultPlugin) OnPrecheckRequest(ctx *PluginContext, number *number.Number) (bool, error) { - return true, nil -} - -func (p *DefaultPlugin) OnHTTPClientInit() HTTPInvoker { - return nil -} - -func (p *DefaultPlugin) OnMakeHTTPRequest(ctx *PluginContext, number *number.Number) (*http.Request, error) { - return nil, fmt.Errorf("no impl") -} - -func (p *DefaultPlugin) OnDecorateRequest(ctx *PluginContext, req *http.Request) error { - return nil -} - -func (p *DefaultPlugin) OnPrecheckResponse(ctx *PluginContext, req *http.Request, rsp *http.Response) (bool, error) { - if rsp.StatusCode == http.StatusNotFound { - return false, nil - } - return true, nil -} - -func (p *DefaultPlugin) OnHandleHTTPRequest(ctx *PluginContext, invoker HTTPInvoker, req *http.Request) (*http.Response, error) { - return invoker(ctx, req) -} - -func (p *DefaultPlugin) OnDecodeHTTPData(ctx *PluginContext, data []byte) (*model.AvMeta, bool, error) { - return nil, false, fmt.Errorf("no impl") -} - -func (p *DefaultPlugin) OnDecorateMediaRequest(ctx *PluginContext, req *http.Request) error { - return nil -} - -type CreatorFunc func(args interface{}) (IPlugin, error) - -var mp = make(map[string]CreatorFunc) - -func Register(name string, fn CreatorFunc) { - mp[name] = fn -} - -func CreatePlugin(name string, args interface{}) (IPlugin, error) { - cr, ok := mp[name] - if !ok { - return nil, fmt.Errorf("plugin:%s not found", name) - } - return cr(args) -} - -func PluginToCreator(plg IPlugin) CreatorFunc { - return func(args interface{}) (IPlugin, error) { - return plg, nil - } -} - -func Plugins() []string { - rs := make([]string, 0, len(mp)) - for k := range mp { - rs = append(rs, k) - } - return sort.StringSlice(rs) -} diff --git a/searcher/plugin/register/register.go b/searcher/plugin/register/register.go new file mode 100644 index 0000000..02a7853 --- /dev/null +++ b/searcher/plugin/register/register.go @@ -0,0 +1,6 @@ +package register + +import ( + _ "yamdc/searcher/plugin/impl" + _ "yamdc/searcher/plugin/impl/airav" +) diff --git a/searcher/plugin/twostep.go b/searcher/plugin/twostep/twostep.go similarity index 90% rename from searcher/plugin/twostep.go rename to searcher/plugin/twostep/twostep.go index 6ea6176..59c8780 100644 --- a/searcher/plugin/twostep.go +++ b/searcher/plugin/twostep/twostep.go @@ -1,9 +1,11 @@ -package plugin +package twostep import ( + "context" "fmt" "net/http" "yamdc/searcher/decoder" + "yamdc/searcher/plugin/api" "yamdc/searcher/utils" ) @@ -32,7 +34,7 @@ func isCodeInValidStatusCodeList(lst []int, code int) bool { return false } -func HandleXPathTwoStepSearch(ctx *PluginContext, invoker HTTPInvoker, req *http.Request, xctx *XPathTwoStepContext) (*http.Response, error) { +func HandleXPathTwoStepSearch(ctx context.Context, invoker api.HTTPInvoker, req *http.Request, xctx *XPathTwoStepContext) (*http.Response, error) { rsp, err := invoker(ctx, req) if err != nil { return nil, fmt.Errorf("step search failed, err:%w", err)