Skip to content

Commit

Permalink
Fix for Reuters 401s
Browse files Browse the repository at this point in the history
  • Loading branch information
earthboundkid committed Dec 6, 2023
1 parent e650630 commit fb3c224
Show file tree
Hide file tree
Showing 2 changed files with 3 additions and 5 deletions.
7 changes: 3 additions & 4 deletions linkcheck/linkcheck.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ var (
)

const (
chromeUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36"
safariUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1.2 Safari/605.1.15"
)

// CLI runs the linkrot executable, equivalent to calling it on the command line.
Expand Down Expand Up @@ -121,7 +121,6 @@ Options:
excludePaths,
logger,
cl,
chromeUserAgent,
*shouldArchive,
}

Expand All @@ -136,7 +135,6 @@ type crawler struct {
excludePaths []string
l *slog.Logger
*http.Client
userAgent string
shouldArchive bool
}

Expand Down Expand Up @@ -229,7 +227,8 @@ func (c *crawler) doFetch(ctx context.Context, pageurl string) (links, ids []str
URL(pageurl).
Method(method).
Accept("text/html,application/xhtml+xml,application/xml,*/*").
UserAgent(c.userAgent).
Header("Accept-Language", "en-US,en;q=0.9").
UserAgent(safariUserAgent).
Client(c.Client).
CheckStatus(http.StatusOK).
CheckContentType(
Expand Down
1 change: 0 additions & 1 deletion linkcheck/linkcheck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ func TestRun(t *testing.T) {
excludePaths,
slog.New(slog.NewTextHandler(io.Discard, nil)),
ts.Client(),
chromeUserAgent,
false,
}

Expand Down

0 comments on commit fb3c224

Please sign in to comment.